source.dussan.org Git - archiva.git/blob

   1 package org.apache.archiva.repository.maven.metadata.storage;
   2
   3 /*
   4  * Licensed to the Apache Software Foundation (ASF) under one
   5  * or more contributor license agreements.  See the NOTICE file
   6  * distributed with this work for additional information
   7  * regarding copyright ownership.  The ASF licenses this file
   8  * to you under the Apache License, Version 2.0 (the
   9  * "License"); you may not use this file except in compliance
  10  * with the License.  You may obtain a copy of the License at
  11  *
  12  * http://www.apache.org/licenses/LICENSE-2.0
  13  * Unless required by applicable law or agreed to in writing,
  14  * software distributed under the License is distributed on an
  15  * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
  16  * KIND, either express or implied.  See the License for the
  17  * specific language governing permissions and limitations
  18  * under the License.
  19  */
  20
  21 import org.apache.archiva.checksum.ChecksumAlgorithm;
  22 import org.apache.archiva.checksum.ChecksummedFile;
  23 import org.apache.archiva.common.Try;
  24 import org.apache.archiva.common.utils.VersionUtil;
  25 import org.apache.archiva.filter.Filter;
  26 import org.apache.archiva.metadata.maven.MavenMetadataReader;
  27 import org.apache.archiva.metadata.model.ArtifactMetadata;
  28 import org.apache.archiva.metadata.model.ProjectMetadata;
  29 import org.apache.archiva.metadata.model.ProjectVersionMetadata;
  30 import org.apache.archiva.metadata.model.facets.RepositoryProblemFacet;
  31 import org.apache.archiva.metadata.repository.storage.*;
  32 import org.apache.archiva.model.ArchivaRepositoryMetadata;
  33 import org.apache.archiva.model.ArtifactReference;
  34 import org.apache.archiva.model.SnapshotVersion;
  35 import org.apache.archiva.policies.ProxyDownloadException;
  36 import org.apache.archiva.proxy.ProxyRegistry;
  37 import org.apache.archiva.proxy.maven.WagonFactory;
  38 import org.apache.archiva.proxy.model.NetworkProxy;
  39 import org.apache.archiva.proxy.model.ProxyConnector;
  40 import org.apache.archiva.proxy.model.RepositoryProxyHandler;
  41 import org.apache.archiva.repository.*;
  42 import org.apache.archiva.repository.content.Artifact;
  43 import org.apache.archiva.repository.content.ContentItem;
  44 import org.apache.archiva.repository.content.ItemSelector;
  45 import org.apache.archiva.repository.content.PathParser;
  46 import org.apache.archiva.repository.content.base.ArchivaItemSelector;
  47 import org.apache.archiva.repository.maven.MavenSystemManager;
  48 import org.apache.archiva.repository.metadata.RepositoryMetadataException;
  49 import org.apache.archiva.repository.storage.StorageAsset;
  50 import org.apache.commons.lang3.ArrayUtils;
  51 import org.apache.commons.lang3.StringUtils;
  52 import org.apache.maven.model.*;
  53 import org.apache.maven.model.building.*;
  54 import org.apache.maven.model.io.xpp3.MavenXpp3Reader;
  55 import org.codehaus.plexus.util.xml.pull.XmlPullParserException;
  56 import org.slf4j.Logger;
  57 import org.slf4j.LoggerFactory;
  58 import org.springframework.context.ApplicationContext;
  59 import org.springframework.stereotype.Service;
  60
  61 import javax.annotation.PostConstruct;
  62 import javax.inject.Inject;
  63 import javax.inject.Named;
  64 import java.io.FileNotFoundException;
  65 import java.io.IOException;
  66 import java.io.Reader;
  67 import java.nio.channels.Channels;
  68 import java.nio.charset.Charset;
  69 import java.nio.file.NoSuchFileException;
  70 import java.time.ZoneId;
  71 import java.time.ZonedDateTime;
  72 import java.util.*;
  73 import java.util.function.Predicate;
  74 import java.util.stream.Collectors;
  75
  76 // import java.io.FileNotFoundException;
  77
  78 /**
  79  * <p>
  80  * Maven 2 repository format storage implementation. This class currently takes parameters to indicate the repository to
  81  * deal with rather than being instantiated per-repository.
  82  * FIXME: instantiate one per repository and allocate permanently from a factory (which can be obtained within the session).
  83  * </p>
  84  * <p>
  85  * The session is passed in as an argument to obtain any necessary resources, rather than the class being instantiated
  86  * within the session in the context of a single managed repository's resolution needs.
  87  * </p>
  88  */
  89 @Service("repositoryStorage#maven2")
  90 public class Maven2RepositoryStorage
  91         implements RepositoryStorage {
  92
  93     private static final Logger log = LoggerFactory.getLogger(Maven2RepositoryStorage.class);
  94
  95     private ModelBuilder builder;
  96
  97     @Inject
  98     RepositoryRegistry repositoryRegistry;
  99
 100     @Inject
 101     @Named( "metadataReader#maven" )
 102     MavenMetadataReader metadataReader;
 103
 104     @Inject
 105     @Named("repositoryPathTranslator#maven2")
 106     private RepositoryPathTranslator pathTranslator;
 107
 108     @Inject
 109     private WagonFactory wagonFactory;
 110
 111     @Inject
 112     private ApplicationContext applicationContext;
 113
 114     @Inject
 115     @Named("pathParser#default")
 116     private PathParser pathParser;
 117
 118     @Inject
 119     private ProxyRegistry proxyRegistry;
 120
 121     @Inject
 122     private MavenSystemManager mavenSystemManager;
 123
 124     private static final String METADATA_FILENAME_START = "maven-metadata";
 125
 126     private static final String METADATA_FILENAME = METADATA_FILENAME_START + ".xml";
 127
 128     // This array must be lexically sorted
 129     private static final String[] IGNORED_FILES = {METADATA_FILENAME, "resolver-status.properties"};
 130
 131     private static final MavenXpp3Reader MAVEN_XPP_3_READER = new MavenXpp3Reader();
 132
 133
 134     @PostConstruct
 135     public void initialize() {
 136         builder = new DefaultModelBuilderFactory().newInstance();
 137
 138     }
 139
 140     @Override
 141     public ProjectMetadata readProjectMetadata(String repoId, String namespace, String projectId) {
 142         // TODO: could natively implement the "shared model" concept from the browse action to avoid needing it there?
 143         return null;
 144     }
 145
 146     @Override
 147     public ProjectVersionMetadata readProjectVersionMetadata(ReadMetadataRequest readMetadataRequest)
 148             throws RepositoryStorageMetadataNotFoundException, RepositoryStorageMetadataInvalidException,
 149             RepositoryStorageRuntimeException {
 150
 151         ManagedRepository managedRepository = repositoryRegistry.getManagedRepository(readMetadataRequest.getRepositoryId());
 152         boolean isReleases = managedRepository.getActiveReleaseSchemes().contains(ReleaseScheme.RELEASE);
 153         boolean isSnapshots = managedRepository.getActiveReleaseSchemes().contains(ReleaseScheme.SNAPSHOT);
 154         String artifactVersion = readMetadataRequest.getProjectVersion();
 155         // olamy: in case of browsing via the ui we can mix repos (parent of a SNAPSHOT can come from release repo)
 156         if (!readMetadataRequest.isBrowsingRequest()) {
 157             if (VersionUtil.isSnapshot(artifactVersion)) {
 158                 // skygo trying to improve speed by honoring managed configuration MRM-1658
 159                 if (isReleases && !isSnapshots) {
 160                     throw new RepositoryStorageRuntimeException("lookforsnaponreleaseonly",
 161                             "managed repo is configured for release only");
 162                 }
 163             } else {
 164                 if (!isReleases && isSnapshots) {
 165                     throw new RepositoryStorageRuntimeException("lookforsreleaseonsneponly",
 166                             "managed repo is configured for snapshot only");
 167                 }
 168             }
 169         }
 170         StorageAsset basedir = managedRepository.getRoot();
 171         if (VersionUtil.isSnapshot(artifactVersion)) {
 172             StorageAsset metadataFile = pathTranslator.toFile(basedir, readMetadataRequest.getNamespace(),
 173                     readMetadataRequest.getProjectId(), artifactVersion,
 174                     METADATA_FILENAME);
 175             try {
 176                 ArchivaRepositoryMetadata metadata = metadataReader.read(metadataFile);
 177
 178                 // re-adjust to timestamp if present, otherwise retain the original -SNAPSHOT filename
 179                 SnapshotVersion snapshotVersion = metadata.getSnapshotVersion();
 180                 if (snapshotVersion != null) {
 181                     artifactVersion =
 182                             artifactVersion.substring(0, artifactVersion.length() - 8); // remove SNAPSHOT from end
 183                     artifactVersion =
 184                             artifactVersion + snapshotVersion.getTimestamp() + "-" + snapshotVersion.getBuildNumber();
 185                 }
 186             } catch ( RepositoryMetadataException e) {
 187                 // unable to parse metadata - LOGGER it, and continue with the version as the original SNAPSHOT version
 188                 log.warn("Invalid metadata: {} - {}", metadataFile, e.getMessage());
 189             }
 190         }
 191
 192         // TODO: won't work well with some other layouts, might need to convert artifact parts to ID by path translator
 193         String id = readMetadataRequest.getProjectId() + "-" + artifactVersion + ".pom";
 194         StorageAsset file =
 195                 pathTranslator.toFile(basedir, readMetadataRequest.getNamespace(), readMetadataRequest.getProjectId(),
 196                         readMetadataRequest.getProjectVersion(), id);
 197
 198         if (!file.exists()) {
 199             // metadata could not be resolved
 200             throw new RepositoryStorageMetadataNotFoundException(
 201                     "The artifact's POM file '" + file.getPath() + "' was missing");
 202         }
 203
 204         // TODO: this is a workaround until we can properly resolve using proxies as well - this doesn't cache
 205         //       anything locally!
 206         List<RemoteRepository> remoteRepositories = new ArrayList<>();
 207         Map<String, NetworkProxy> networkProxies = new HashMap<>();
 208
 209         Map<String, List<ProxyConnector>> proxyConnectorsMap = proxyRegistry.getProxyConnectorAsMap();
 210         List<ProxyConnector> proxyConnectors = proxyConnectorsMap.get(readMetadataRequest.getRepositoryId());
 211         if (proxyConnectors != null) {
 212             for (ProxyConnector proxyConnector : proxyConnectors) {
 213                 RemoteRepository remoteRepoConfig =
 214                         repositoryRegistry.getRemoteRepository(proxyConnector.getTargetRepository().getId());
 215
 216                 if (remoteRepoConfig != null) {
 217                     remoteRepositories.add(remoteRepoConfig);
 218
 219                     NetworkProxy networkProxyConfig =
 220                             proxyRegistry.getNetworkProxy(proxyConnector.getProxyId());
 221
 222                     if (networkProxyConfig != null) {
 223                         // key/value: remote repo ID/proxy info
 224                         networkProxies.put(proxyConnector.getTargetRepository().getId(), networkProxyConfig);
 225                     }
 226                 }
 227             }
 228         }
 229
 230         // That's a browsing request so we can a mix of SNAPSHOT and release artifacts (especially with snapshots which
 231         // can have released parent pom
 232         if (readMetadataRequest.isBrowsingRequest()) {
 233             remoteRepositories.addAll(repositoryRegistry.getRemoteRepositories());
 234         }
 235
 236         ModelBuildingRequest req =
 237                 new DefaultModelBuildingRequest().setProcessPlugins(false).setPomFile(file.getFilePath().toFile()).setTwoPhaseBuilding(
 238                         false).setValidationLevel(ModelBuildingRequest.VALIDATION_LEVEL_MINIMAL);
 239
 240         //MRM-1607. olamy this will resolve jdk profiles on the current running archiva jvm
 241         req.setSystemProperties(System.getProperties());
 242
 243         // MRM-1411
 244         req.setModelResolver(
 245                 new RepositoryModelResolver(managedRepository, pathTranslator, wagonFactory, remoteRepositories,
 246                         networkProxies, managedRepository, mavenSystemManager, metadataReader));
 247
 248         Model model;
 249         try {
 250             model = builder.build(req).getEffectiveModel();
 251         } catch (ModelBuildingException e) {
 252             String msg = "The artifact's POM file '" + file + "' was invalid: " + e.getMessage();
 253
 254             List<ModelProblem> modelProblems = e.getProblems();
 255             for (ModelProblem problem : modelProblems) {
 256                 // MRM-1411, related to MRM-1335
 257                 // this means that the problem was that the parent wasn't resolved!
 258                 // olamy really hackhish but fail with java profile so use error message
 259                 // || ( StringUtils.startsWith( problem.getMessage(), "Failed to determine Java version for profile" ) )
 260                 // but setTwoPhaseBuilding(true) fix that
 261                 if (((problem.getException() instanceof FileNotFoundException
 262                         || problem.getException() instanceof NoSuchFileException
 263                 ) && e.getModelId() != null &&
 264                         !e.getModelId().equals(problem.getModelId()))) {
 265                     log.warn("The artifact's parent POM file '{}' cannot be resolved. "
 266                             + "Using defaults for project version metadata..", file);
 267
 268                     ProjectVersionMetadata metadata = new ProjectVersionMetadata();
 269                     metadata.setId(readMetadataRequest.getProjectVersion());
 270
 271                     MavenProjectFacet facet = new MavenProjectFacet();
 272                     facet.setGroupId(readMetadataRequest.getNamespace());
 273                     facet.setArtifactId(readMetadataRequest.getProjectId());
 274                     facet.setPackaging("jar");
 275                     metadata.addFacet(facet);
 276
 277                     String errMsg =
 278                             "Error in resolving artifact's parent POM file. " + (problem.getException() == null
 279                                     ? problem.getMessage()
 280                                     : problem.getException().getMessage());
 281                     RepositoryProblemFacet repoProblemFacet = new RepositoryProblemFacet();
 282                     repoProblemFacet.setRepositoryId(readMetadataRequest.getRepositoryId());
 283                     repoProblemFacet.setId(readMetadataRequest.getRepositoryId());
 284                     repoProblemFacet.setMessage(errMsg);
 285                     repoProblemFacet.setProblem(errMsg);
 286                     repoProblemFacet.setProject(readMetadataRequest.getProjectId());
 287                     repoProblemFacet.setVersion(readMetadataRequest.getProjectVersion());
 288                     repoProblemFacet.setNamespace(readMetadataRequest.getNamespace());
 289
 290                     metadata.addFacet(repoProblemFacet);
 291
 292                     return metadata;
 293                 }
 294             }
 295
 296             throw new RepositoryStorageMetadataInvalidException("invalid-pom", msg, e);
 297         }
 298
 299         // Check if the POM is in the correct location
 300         boolean correctGroupId = readMetadataRequest.getNamespace().equals(model.getGroupId());
 301         boolean correctArtifactId = readMetadataRequest.getProjectId().equals(model.getArtifactId());
 302         boolean correctVersion = readMetadataRequest.getProjectVersion().equals(model.getVersion());
 303         if (!correctGroupId || !correctArtifactId || !correctVersion) {
 304             StringBuilder message = new StringBuilder("Incorrect POM coordinates in '" + file + "':");
 305             if (!correctGroupId) {
 306                 message.append("\nIncorrect group ID: ").append(model.getGroupId());
 307             }
 308             if (!correctArtifactId) {
 309                 message.append("\nIncorrect artifact ID: ").append(model.getArtifactId());
 310             }
 311             if (!correctVersion) {
 312                 message.append("\nIncorrect version: ").append(model.getVersion());
 313             }
 314
 315             throw new RepositoryStorageMetadataInvalidException("mislocated-pom", message.toString());
 316         }
 317
 318         ProjectVersionMetadata metadata = new ProjectVersionMetadata();
 319         metadata.setCiManagement(convertCiManagement(model.getCiManagement()));
 320         metadata.setDescription(model.getDescription());
 321         metadata.setId(readMetadataRequest.getProjectVersion());
 322         metadata.setIssueManagement(convertIssueManagement(model.getIssueManagement()));
 323         metadata.setLicenses(convertLicenses(model.getLicenses()));
 324         metadata.setMailingLists(convertMailingLists(model.getMailingLists()));
 325         metadata.setDependencies(convertDependencies(model.getDependencies()));
 326         metadata.setName(model.getName());
 327         metadata.setOrganization(convertOrganization(model.getOrganization()));
 328         metadata.setScm(convertScm(model.getScm()));
 329         metadata.setUrl(model.getUrl());
 330         metadata.setProperties(model.getProperties());
 331
 332         MavenProjectFacet facet = new MavenProjectFacet();
 333         facet.setGroupId(model.getGroupId() != null ? model.getGroupId() : model.getParent().getGroupId());
 334         facet.setArtifactId(model.getArtifactId());
 335         facet.setPackaging(model.getPackaging());
 336         if (model.getParent() != null) {
 337             MavenProjectParent parent = new MavenProjectParent();
 338             parent.setGroupId(model.getParent().getGroupId());
 339             parent.setArtifactId(model.getParent().getArtifactId());
 340             parent.setVersion(model.getParent().getVersion());
 341             facet.setParent(parent);
 342         }
 343         metadata.addFacet(facet);
 344
 345         return metadata;
 346
 347
 348     }
 349
 350     public void setWagonFactory(WagonFactory wagonFactory) {
 351         this.wagonFactory = wagonFactory;
 352     }
 353
 354     private List<org.apache.archiva.metadata.model.Dependency> convertDependencies(List<Dependency> dependencies) {
 355         List<org.apache.archiva.metadata.model.Dependency> l = new ArrayList<>();
 356         for (Dependency dependency : dependencies) {
 357             org.apache.archiva.metadata.model.Dependency newDependency =
 358                     new org.apache.archiva.metadata.model.Dependency();
 359             newDependency.setArtifactId(dependency.getArtifactId());
 360             newDependency.setClassifier(dependency.getClassifier());
 361             newDependency.setNamespace(dependency.getGroupId());
 362             newDependency.setOptional(dependency.isOptional());
 363             newDependency.setScope(dependency.getScope());
 364             newDependency.setSystemPath(dependency.getSystemPath());
 365             newDependency.setType(dependency.getType());
 366             newDependency.setVersion(dependency.getVersion());
 367             l.add(newDependency);
 368         }
 369         return l;
 370     }
 371
 372     private org.apache.archiva.metadata.model.Scm convertScm(Scm scm) {
 373         org.apache.archiva.metadata.model.Scm newScm = null;
 374         if (scm != null) {
 375             newScm = new org.apache.archiva.metadata.model.Scm();
 376             newScm.setConnection(scm.getConnection());
 377             newScm.setDeveloperConnection(scm.getDeveloperConnection());
 378             newScm.setUrl(scm.getUrl());
 379         }
 380         return newScm;
 381     }
 382
 383     private org.apache.archiva.metadata.model.Organization convertOrganization(Organization organization) {
 384         org.apache.archiva.metadata.model.Organization org = null;
 385         if (organization != null) {
 386             org = new org.apache.archiva.metadata.model.Organization();
 387             org.setName(organization.getName());
 388             org.setUrl(organization.getUrl());
 389         }
 390         return org;
 391     }
 392
 393     private List<org.apache.archiva.metadata.model.License> convertLicenses(List<License> licenses) {
 394         List<org.apache.archiva.metadata.model.License> l = new ArrayList<>();
 395         for (License license : licenses) {
 396             org.apache.archiva.metadata.model.License newLicense = new org.apache.archiva.metadata.model.License();
 397             newLicense.setName(license.getName());
 398             newLicense.setUrl(license.getUrl());
 399             l.add(newLicense);
 400         }
 401         return l;
 402     }
 403
 404     private List<org.apache.archiva.metadata.model.MailingList> convertMailingLists(List<MailingList> mailingLists) {
 405         List<org.apache.archiva.metadata.model.MailingList> l = new ArrayList<>();
 406         for (MailingList mailingList : mailingLists) {
 407             org.apache.archiva.metadata.model.MailingList newMailingList =
 408                     new org.apache.archiva.metadata.model.MailingList();
 409             newMailingList.setName(mailingList.getName());
 410             newMailingList.setMainArchiveUrl(mailingList.getArchive());
 411             newMailingList.setPostAddress(mailingList.getPost());
 412             newMailingList.setSubscribeAddress(mailingList.getSubscribe());
 413             newMailingList.setUnsubscribeAddress(mailingList.getUnsubscribe());
 414             newMailingList.setOtherArchives(mailingList.getOtherArchives());
 415             l.add(newMailingList);
 416         }
 417         return l;
 418     }
 419
 420     private org.apache.archiva.metadata.model.IssueManagement convertIssueManagement(IssueManagement issueManagement) {
 421         org.apache.archiva.metadata.model.IssueManagement im = null;
 422         if (issueManagement != null) {
 423             im = new org.apache.archiva.metadata.model.IssueManagement();
 424             im.setSystem(issueManagement.getSystem());
 425             im.setUrl(issueManagement.getUrl());
 426         }
 427         return im;
 428     }
 429
 430     private org.apache.archiva.metadata.model.CiManagement convertCiManagement(CiManagement ciManagement) {
 431         org.apache.archiva.metadata.model.CiManagement ci = null;
 432         if (ciManagement != null) {
 433             ci = new org.apache.archiva.metadata.model.CiManagement();
 434             ci.setSystem(ciManagement.getSystem());
 435             ci.setUrl(ciManagement.getUrl());
 436         }
 437         return ci;
 438     }
 439
 440     @Override
 441     public Collection<String> listRootNamespaces(String repoId, Filter<String> filter)
 442             throws RepositoryStorageRuntimeException {
 443         StorageAsset dir = getRepositoryBasedir(repoId);
 444
 445         return getSortedFiles(dir, filter);
 446     }
 447
 448     private static Collection<String> getSortedFiles(StorageAsset dir, Filter<String> filter) {
 449
 450         final Predicate<StorageAsset> dFilter = new DirectoryFilter(filter);
 451         return dir.list().stream().filter(f -> f.isContainer())
 452                 .filter(dFilter)
 453                 .map(path -> path.getName().toString())
 454                 .sorted().collect(Collectors.toList());
 455
 456     }
 457
 458     private StorageAsset getRepositoryBasedir(String repoId)
 459             throws RepositoryStorageRuntimeException {
 460         ManagedRepository repository = repositoryRegistry.getManagedRepository(repoId);
 461
 462         return repository.getRoot();
 463     }
 464
 465     @Override
 466     public Collection<String> listNamespaces(String repoId, String namespace, Filter<String> filter)
 467             throws RepositoryStorageRuntimeException {
 468         StorageAsset dir = pathTranslator.toFile(getRepositoryBasedir(repoId), namespace);
 469         if (!(dir.exists()) && !dir.isContainer()) {
 470             return Collections.emptyList();
 471         }
 472         // scan all the directories which are potential namespaces. Any directories known to be projects are excluded
 473         Predicate<StorageAsset> dFilter = new DirectoryFilter(filter);
 474         return dir.list().stream().filter(dFilter).filter(path -> !isProject(path, filter)).map(path -> path.getName().toString())
 475                 .sorted().collect(Collectors.toList());
 476     }
 477
 478     @Override
 479     public Collection<String> listProjects(String repoId, String namespace, Filter<String> filter)
 480             throws RepositoryStorageRuntimeException {
 481         StorageAsset dir = pathTranslator.toFile(getRepositoryBasedir(repoId), namespace);
 482         if (!(dir.exists() && dir.isContainer())) {
 483             return Collections.emptyList();
 484         }
 485         // scan all directories in the namespace, and only include those that are known to be projects
 486         final Predicate<StorageAsset> dFilter = new DirectoryFilter(filter);
 487         return dir.list().stream().filter(dFilter).filter(path -> isProject(path, filter)).map(path -> path.getName().toString())
 488                 .sorted().collect(Collectors.toList());
 489
 490     }
 491
 492     @Override
 493     public Collection<String> listProjectVersions(String repoId, String namespace, String projectId,
 494                                                   Filter<String> filter)
 495             throws RepositoryStorageRuntimeException {
 496         StorageAsset dir = pathTranslator.toFile(getRepositoryBasedir(repoId), namespace, projectId);
 497         if (!(dir.exists() && dir.isContainer())) {
 498             return Collections.emptyList();
 499         }
 500
 501         // all directories in a project directory can be considered a version
 502         return getSortedFiles(dir, filter);
 503     }
 504
 505     @Override
 506     public Collection<ArtifactMetadata> readArtifactsMetadata(ReadMetadataRequest readMetadataRequest)
 507             throws RepositoryStorageRuntimeException {
 508         StorageAsset dir = pathTranslator.toFile(getRepositoryBasedir(readMetadataRequest.getRepositoryId()),
 509                 readMetadataRequest.getNamespace(), readMetadataRequest.getProjectId(),
 510                 readMetadataRequest.getProjectVersion());
 511         if (!(dir.exists() && dir.isContainer())) {
 512             return Collections.emptyList();
 513         }
 514
 515         // all files that are not metadata and not a checksum / signature are considered artifacts
 516         final Predicate<StorageAsset> dFilter = new ArtifactDirectoryFilter(readMetadataRequest.getFilter());
 517         // Returns a map TRUE -> (success values), FALSE -> (Exceptions)
 518         Map<Boolean, List<Try<ArtifactMetadata>>> result = dir.list().stream().filter(dFilter).map(path -> {
 519                     try {
 520                         return Try.success(getArtifactFromFile(readMetadataRequest.getRepositoryId(), readMetadataRequest.getNamespace(),
 521                                 readMetadataRequest.getProjectId(), readMetadataRequest.getProjectVersion(),
 522                                 path));
 523                     } catch (Exception e) {
 524                         log.debug("Could not create metadata for {}:  {}", path, e.getMessage(), e);
 525                         return Try.<ArtifactMetadata>failure(e);
 526                     }
 527                 }
 528         ).collect(Collectors.groupingBy(Try::isSuccess));
 529         if (result.containsKey(Boolean.FALSE) && result.get(Boolean.FALSE).size() > 0 && (!result.containsKey(Boolean.TRUE) || result.get(Boolean.TRUE).size() == 0)) {
 530             log.error("Could not get artifact metadata. Directory: {}. Number of errors {}.", dir, result.get(Boolean.FALSE).size());
 531             Try<ArtifactMetadata> failure = result.get(Boolean.FALSE).get(0);
 532             log.error("Sample exception {}", failure.getError().getMessage(), failure.getError());
 533             throw new RepositoryStorageRuntimeException(readMetadataRequest.getRepositoryId(), "Could not retrieve metadata of the files");
 534         } else {
 535             if (!result.containsKey(Boolean.TRUE) || result.get(Boolean.TRUE) == null) {
 536                 return Collections.emptyList();
 537             }
 538             return result.get(Boolean.TRUE).stream().map(tr -> tr.get()).collect(Collectors.toList());
 539         }
 540
 541     }
 542
 543     @Override
 544     public ArtifactMetadata readArtifactMetadataFromPath(String repoId, String path)
 545             throws RepositoryStorageRuntimeException {
 546         ArtifactMetadata metadata = pathTranslator.getArtifactForPath(repoId, path);
 547
 548         try {
 549             populateArtifactMetadataFromFile(metadata, getRepositoryBasedir(repoId).resolve(path));
 550         } catch (IOException e) {
 551             throw new RepositoryStorageRuntimeException(repoId, "Error during metadata retrieval of " + path + " :" + e.getMessage(), e);
 552         }
 553
 554         return metadata;
 555     }
 556
 557     private ArtifactMetadata getArtifactFromFile(String repoId, String namespace, String projectId,
 558                                                  String projectVersion, StorageAsset file) throws IOException {
 559         ArtifactMetadata metadata =
 560                 pathTranslator.getArtifactFromId(repoId, namespace, projectId, projectVersion, file.getName());
 561
 562         populateArtifactMetadataFromFile(metadata, file);
 563
 564         return metadata;
 565     }
 566
 567     @Override
 568     public void applyServerSideRelocation(ManagedRepository managedRepository, ArtifactReference artifact)
 569             throws ProxyDownloadException {
 570         if ("pom".equals(artifact.getType())) {
 571             return;
 572         }
 573
 574         // Build the artifact POM reference
 575         ArtifactReference pomReference = new ArtifactReference();
 576         pomReference.setGroupId(artifact.getGroupId());
 577         pomReference.setArtifactId(artifact.getArtifactId());
 578         pomReference.setVersion(artifact.getVersion());
 579         pomReference.setProjectVersion( artifact.getProjectVersion() );
 580         pomReference.setType("pom");
 581         BaseRepositoryContentLayout layout;
 582         try
 583         {
 584             layout = managedRepository.getContent( ).getLayout( BaseRepositoryContentLayout.class );
 585         }
 586         catch ( LayoutException e )
 587         {
 588             throw new ProxyDownloadException( "Could not set layout " + e.getMessage( ), new HashMap<>(  ) );
 589         }
 590
 591         RepositoryType repositoryType = managedRepository.getType();
 592         if (!proxyRegistry.hasHandler(repositoryType)) {
 593             throw new ProxyDownloadException("No proxy handler found for repository type " + repositoryType, new HashMap<>());
 594         }
 595
 596         ItemSelector selector = ArchivaItemSelector.builder( )
 597             .withNamespace( artifact.getGroupId( ) )
 598             .withProjectId( artifact.getArtifactId( ) )
 599             .withArtifactId( artifact.getArtifactId( ) )
 600             .withVersion( artifact.getVersion( ) )
 601             .withArtifactVersion( artifact.getVersion( ) )
 602             .withType( "pom" ).build( );
 603
 604         Artifact pom = layout.getArtifact( selector );
 605
 606         RepositoryProxyHandler proxyHandler = proxyRegistry.getHandler(repositoryType).get(0);
 607
 608         // Get the artifact POM from proxied repositories if needed
 609         proxyHandler.fetchFromProxies(managedRepository, pomReference);
 610
 611         // Open and read the POM from the managed repo
 612
 613         if (!pom.exists()) {
 614             return;
 615         }
 616
 617         try {
 618             // MavenXpp3Reader leaves the file open, so we need to close it ourselves.
 619
 620             Model model;
 621             try (Reader reader = Channels.newReader(pom.getAsset().getReadChannel(), Charset.defaultCharset().name())) {
 622                 model = MAVEN_XPP_3_READER.read(reader);
 623             }
 624
 625             DistributionManagement dist = model.getDistributionManagement();
 626             if (dist != null) {
 627                 Relocation relocation = dist.getRelocation();
 628                 if (relocation != null) {
 629                     // artifact is relocated : update the repositoryPath
 630                     if (relocation.getGroupId() != null) {
 631                         artifact.setGroupId(relocation.getGroupId());
 632                     }
 633                     if (relocation.getArtifactId() != null) {
 634                         artifact.setArtifactId(relocation.getArtifactId());
 635                     }
 636                     if (relocation.getVersion() != null) {
 637                         artifact.setVersion(relocation.getVersion());
 638                     }
 639                 }
 640             }
 641         } catch (IOException e) {
 642             // Unable to read POM : ignore.
 643         } catch (XmlPullParserException e) {
 644             // Invalid POM : ignore
 645         }
 646     }
 647
 648     @Override
 649     public ItemSelector applyServerSideRelocation(ManagedRepository managedRepository, ItemSelector artifactSelector)
 650         throws ProxyDownloadException {
 651         if ("pom".equals(artifactSelector.getType())) {
 652             return artifactSelector;
 653         }
 654
 655         // Build the artifact POM reference
 656         BaseRepositoryContentLayout layout;
 657         try
 658         {
 659             layout = managedRepository.getContent( ).getLayout( BaseRepositoryContentLayout.class );
 660         }
 661         catch ( LayoutException e )
 662         {
 663             throw new ProxyDownloadException( "Could not set layout " + e.getMessage( ), new HashMap<>(  ) );
 664         }
 665
 666         RepositoryType repositoryType = managedRepository.getType();
 667         if (!proxyRegistry.hasHandler(repositoryType)) {
 668             throw new ProxyDownloadException("No proxy handler found for repository type " + repositoryType, new HashMap<>());
 669         }
 670
 671
 672
 673         ItemSelector selector = ArchivaItemSelector.builder( )
 674             .withNamespace( artifactSelector.getNamespace( ) )
 675             .withProjectId( artifactSelector.getArtifactId( ) )
 676             .withArtifactId( artifactSelector.getArtifactId( ) )
 677             .withVersion( artifactSelector.getVersion( ) )
 678             .withArtifactVersion( artifactSelector.getVersion( ) )
 679             .withType( "pom" ).build( );
 680
 681         Artifact pom = layout.getArtifact( selector );
 682
 683         RepositoryProxyHandler proxyHandler = proxyRegistry.getHandler(repositoryType).get(0);
 684
 685         // Get the artifact POM from proxied repositories if needed
 686         proxyHandler.fetchFromProxies(managedRepository, pom);
 687
 688         // Open and read the POM from the managed repo
 689
 690         if (!pom.exists()) {
 691             return artifactSelector;
 692         }
 693
 694         try {
 695             // MavenXpp3Reader leaves the file open, so we need to close it ourselves.
 696
 697             Model model;
 698             try (Reader reader = Channels.newReader(pom.getAsset().getReadChannel(), Charset.defaultCharset().name())) {
 699                 model = MAVEN_XPP_3_READER.read(reader);
 700             }
 701
 702             DistributionManagement dist = model.getDistributionManagement();
 703             if (dist != null) {
 704                 Relocation relocation = dist.getRelocation();
 705                 if (relocation != null) {
 706                     ArchivaItemSelector.Builder relocatedBuilder = ArchivaItemSelector.builder( );
 707                     // artifact is relocated : update the repositoryPath
 708                     if (relocation.getGroupId() != null) {
 709                         relocatedBuilder.withNamespace( relocation.getGroupId( ) );
 710                     } else {
 711                         relocatedBuilder.withNamespace( artifactSelector.getNamespace( ) );
 712                     }
 713                     if (relocation.getArtifactId() != null) {
 714                         relocatedBuilder.withArtifactId( relocation.getArtifactId( ) );
 715                     } else {
 716                         relocatedBuilder.withArtifactId( artifactSelector.getArtifactId( ) );
 717                     }
 718                     if (relocation.getVersion() != null)
 719                     {
 720                         relocatedBuilder.withVersion( relocation.getVersion( ) );
 721                     } else {
 722                         relocatedBuilder.withVersion( artifactSelector.getVersion( ) );
 723                     }
 724                     return relocatedBuilder.withArtifactVersion( artifactSelector.getArtifactVersion( ) )
 725                         .withClassifier( artifactSelector.getClassifier( ) )
 726                         .withType( artifactSelector.getType( ) )
 727                         .withProjectId( artifactSelector.getProjectId( ) )
 728                         .withExtension( artifactSelector.getExtension( ) )
 729                         .build( );
 730                 }
 731             }
 732         } catch (IOException e) {
 733             // Unable to read POM : ignore.
 734         } catch (XmlPullParserException e) {
 735             // Invalid POM : ignore
 736         }
 737         return artifactSelector;
 738     }
 739
 740
 741     @Override
 742     public String getFilePath(String requestPath, org.apache.archiva.repository.ManagedRepository managedRepository) {
 743         // managedRepository can be null
 744         // extract artifact reference from url
 745         // groupId:artifactId:version:packaging:classifier
 746         //org/apache/archiva/archiva-checksum/1.4-M4-SNAPSHOT/archiva-checksum-1.4-M4-SNAPSHOT.jar
 747         String logicalResource = null;
 748         String requestPathInfo = StringUtils.defaultString(requestPath);
 749
 750         //remove prefix ie /repository/blah becomes /blah
 751         requestPathInfo = removePrefix(requestPathInfo);
 752
 753         // Remove prefixing slash as the repository id doesn't contain it;
 754         if (requestPathInfo.startsWith("/")) {
 755             requestPathInfo = requestPathInfo.substring(1);
 756         }
 757
 758         int slash = requestPathInfo.indexOf('/');
 759         if (slash > 0) {
 760             logicalResource = requestPathInfo.substring(slash);
 761
 762             if (logicalResource.endsWith("/..")) {
 763                 logicalResource += "/";
 764             }
 765
 766             if (logicalResource != null && logicalResource.startsWith("//")) {
 767                 logicalResource = logicalResource.substring(1);
 768             }
 769
 770             if (logicalResource == null) {
 771                 logicalResource = "/";
 772             }
 773         } else {
 774             logicalResource = "/";
 775         }
 776         return logicalResource;
 777
 778     }
 779
 780     @Override
 781     public String getFilePathWithVersion(final String requestPath, ManagedRepositoryContent managedRepositoryContent)
 782             throws RelocationException
 783     {
 784
 785         if (StringUtils.endsWith(requestPath, METADATA_FILENAME)) {
 786             return getFilePath(requestPath, managedRepositoryContent.getRepository());
 787         }
 788
 789         String filePath = getFilePath(requestPath, managedRepositoryContent.getRepository());
 790         Artifact artifact;
 791         try
 792         {
 793             ContentItem item = managedRepositoryContent.toItem( filePath );
 794             artifact = managedRepositoryContent.getLayout( BaseRepositoryContentLayout.class ).adaptItem( Artifact.class, item );
 795         }
 796         catch ( LayoutException e )
 797         {
 798             return filePath;
 799         }
 800
 801         if (VersionUtil.isGenericSnapshot(artifact.getArtifactVersion())) {
 802             // read maven metadata to get last timestamp
 803             StorageAsset metadataDir = managedRepositoryContent.getRepository().getAsset(filePath).getParent();
 804             if (!metadataDir.exists()) {
 805                 return filePath;
 806             }
 807             StorageAsset metadataFile = metadataDir.resolve(METADATA_FILENAME);
 808             if (!metadataFile.exists()) {
 809                 return filePath;
 810             }
 811             ArchivaRepositoryMetadata archivaRepositoryMetadata = null;
 812             try
 813             {
 814                 archivaRepositoryMetadata = metadataReader.read(metadataFile);
 815             }
 816             catch ( RepositoryMetadataException e )
 817             {
 818                 log.error( "Could not read metadata {}", e.getMessage( ), e );
 819                 return filePath;
 820             }
 821             int buildNumber = archivaRepositoryMetadata.getSnapshotVersion().getBuildNumber();
 822             String timestamp = archivaRepositoryMetadata.getSnapshotVersion().getTimestamp();
 823
 824             // MRM-1846
 825             if (buildNumber < 1 && timestamp == null) {
 826                 return filePath;
 827             }
 828
 829             // org/apache/archiva/archiva-checksum/1.4-M4-SNAPSHOT/archiva-checksum-1.4-M4-SNAPSHOT.jar
 830             // ->  archiva-checksum-1.4-M4-20130425.081822-1.jar
 831
 832             filePath = StringUtils.replace(filePath, //
 833                     artifact.getId() //
 834                             + "-" + artifact.getVersion().getId(), //
 835                     artifact.getId() //
 836                             + "-" + StringUtils.remove(artifact.getArtifactVersion(),
 837                             "-" + VersionUtil.SNAPSHOT) //
 838                             + "-" + timestamp //
 839                             + "-" + buildNumber);
 840
 841             throw new RelocationException("/repository/" + managedRepositoryContent.getRepository().getId() +
 842                     (StringUtils.startsWith(filePath, "/") ? "" : "/") + filePath,
 843                     RelocationException.RelocationType.TEMPORARY);
 844
 845         }
 846
 847         return filePath;
 848     }
 849
 850     //-----------------------------
 851     // internal
 852     //-----------------------------
 853
 854     /**
 855      * FIXME remove
 856      *
 857      * @param href
 858      * @return
 859      */
 860     private static String removePrefix(final String href) {
 861         String[] parts = StringUtils.split(href, '/');
 862         parts = (String[]) ArrayUtils.subarray(parts, 1, parts.length);
 863         if (parts == null || parts.length == 0) {
 864             return "/";
 865         }
 866
 867         String joinedString = StringUtils.join(parts, '/');
 868         if (href.endsWith("/")) {
 869             joinedString = joinedString + "/";
 870         }
 871
 872         return joinedString;
 873     }
 874
 875     private static void populateArtifactMetadataFromFile(ArtifactMetadata metadata, StorageAsset file) throws IOException {
 876         metadata.setWhenGathered(ZonedDateTime.now(ZoneId.of("GMT")));
 877         metadata.setFileLastModified(file.getModificationTime().toEpochMilli());
 878         ChecksummedFile checksummedFile = new ChecksummedFile(file.getFilePath());
 879         try {
 880             metadata.setMd5(checksummedFile.calculateChecksum(ChecksumAlgorithm.MD5));
 881         } catch (IOException e) {
 882             log.error("Unable to checksum file {}: {},MD5", file, e.getMessage());
 883         }
 884         try {
 885             metadata.setSha1(checksummedFile.calculateChecksum(ChecksumAlgorithm.SHA1));
 886         } catch (IOException e) {
 887             log.error("Unable to checksum file {}: {},SHA1", file, e.getMessage());
 888         }
 889         metadata.setSize(file.getSize());
 890     }
 891
 892     private boolean isProject(StorageAsset dir, Filter<String> filter) {
 893         // scan directories for a valid project version subdirectory, meaning this must be a project directory
 894         final Predicate<StorageAsset> dFilter = new DirectoryFilter(filter);
 895         boolean projFound = dir.list().stream().filter(dFilter)
 896                 .anyMatch(path -> isProjectVersion(path));
 897         if (projFound) {
 898             return true;
 899         }
 900
 901         // if a metadata file is present, check if this is the "artifactId" directory, marking it as a project
 902         ArchivaRepositoryMetadata metadata = readMetadata(dir);
 903         if (metadata != null && dir.getName().toString().equals(metadata.getArtifactId())) {
 904             return true;
 905         }
 906
 907         return false;
 908     }
 909
 910     private boolean isProjectVersion(StorageAsset dir) {
 911         final String artifactId = dir.getParent().getName();
 912         final String projectVersion = dir.getName();
 913
 914         // check if there is a POM artifact file to ensure it is a version directory
 915
 916         Predicate<StorageAsset> filter;
 917         if (VersionUtil.isSnapshot(projectVersion)) {
 918             filter = new PomFilenameFilter(artifactId, projectVersion);
 919         } else {
 920             final String pomFile = artifactId + "-" + projectVersion + ".pom";
 921             filter = new PomFileFilter(pomFile);
 922         }
 923         if (dir.list().stream().filter(f -> !f.isContainer()).anyMatch(filter)) {
 924             return true;
 925         }
 926         // if a metadata file is present, check if this is the "version" directory, marking it as a project version
 927         ArchivaRepositoryMetadata metadata = readMetadata(dir);
 928         if (metadata != null && projectVersion.equals(metadata.getVersion())) {
 929             return true;
 930         }
 931
 932         return false;
 933     }
 934
 935     private ArchivaRepositoryMetadata readMetadata(StorageAsset directory) {
 936         ArchivaRepositoryMetadata metadata = null;
 937         StorageAsset metadataFile = directory.resolve(METADATA_FILENAME);
 938         if (metadataFile.exists()) {
 939             try {
 940                 metadata = metadataReader.read(metadataFile);
 941             } catch ( RepositoryMetadataException e )
 942             {
 943                 // Ignore missing or invalid metadata
 944             }
 945         }
 946         return metadata;
 947     }
 948
 949     private static class DirectoryFilter
 950             implements Predicate<StorageAsset> {
 951         private final Filter<String> filter;
 952
 953         public DirectoryFilter(Filter<String> filter) {
 954             this.filter = filter;
 955         }
 956
 957         @Override
 958         public boolean test(StorageAsset dir) {
 959             final String name = dir.getName();
 960             if (!filter.accept(name)) {
 961                 return false;
 962             } else if (name.startsWith(".")) {
 963                 return false;
 964             } else if (!dir.isContainer()) {
 965                 return false;
 966             }
 967             return true;
 968         }
 969     }
 970
 971     private static class ArtifactDirectoryFilter
 972             implements Predicate<StorageAsset> {
 973         private final Filter<String> filter;
 974
 975         private ArtifactDirectoryFilter(Filter<String> filter) {
 976             this.filter = filter;
 977         }
 978
 979         @Override
 980         public boolean test(StorageAsset file) {
 981             final Set<String> checksumExts = ChecksumAlgorithm.getAllExtensions();
 982             final String path = file.getPath();
 983             final String name = file.getName();
 984             final String extension = StringUtils.substringAfterLast(name, ".").toLowerCase();
 985             // TODO compare to logic in maven-repository-layer
 986             if (file.isContainer()) {
 987                 return false;
 988             } else if (!filter.accept(name)) {
 989                 return false;
 990             } else if (name.startsWith(".") || path.contains("/.") ) {
 991                 return false;
 992             } else if (checksumExts.contains(extension)) {
 993                 return false;
 994             } else if (Arrays.binarySearch(IGNORED_FILES, name) >= 0) {
 995                 return false;
 996             }
 997             // some files from remote repositories can have name like maven-metadata-archiva-vm-all-public.xml
 998             else if (StringUtils.startsWith(name, METADATA_FILENAME_START) && StringUtils.endsWith(name, ".xml")) {
 999                 return false;
1000             }
1001
1002             return true;
1003
1004         }
1005     }
1006
1007
1008     private static final class PomFilenameFilter
1009             implements Predicate<StorageAsset> {
1010
1011         private final String artifactId, projectVersion;
1012
1013         private PomFilenameFilter(String artifactId, String projectVersion) {
1014             this.artifactId = artifactId;
1015             this.projectVersion = projectVersion;
1016         }
1017
1018         @Override
1019         public boolean test(StorageAsset dir) {
1020             final String name = dir.getName();
1021             if (name.startsWith(artifactId + "-") && name.endsWith(".pom")) {
1022                 String v = name.substring(artifactId.length() + 1, name.length() - 4);
1023                 v = VersionUtil.getBaseVersion(v);
1024                 if (v.equals(projectVersion)) {
1025                     return true;
1026                 }
1027             }
1028             return false;
1029         }
1030
1031     }
1032
1033     private static class PomFileFilter
1034             implements Predicate<StorageAsset> {
1035         private final String pomFile;
1036
1037         private PomFileFilter(String pomFile) {
1038             this.pomFile = pomFile;
1039         }
1040
1041         @Override
1042         public boolean test(StorageAsset dir) {
1043             return pomFile.equals(dir.getName());
1044         }
1045     }
1046
1047
1048     public PathParser getPathParser() {
1049         return pathParser;
1050     }
1051
1052     public void setPathParser(PathParser pathParser) {
1053         this.pathParser = pathParser;
1054     }
1055 }