source.dussan.org Git - archiva.git/blob

   1 package org.apache.archiva.repository.maven.metadata.storage;
   2
   3 /*
   4  * Licensed to the Apache Software Foundation (ASF) under one
   5  * or more contributor license agreements.  See the NOTICE file
   6  * distributed with this work for additional information
   7  * regarding copyright ownership.  The ASF licenses this file
   8  * to you under the Apache License, Version 2.0 (the
   9  * "License"); you may not use this file except in compliance
  10  * with the License.  You may obtain a copy of the License at
  11  *
  12  * http://www.apache.org/licenses/LICENSE-2.0
  13  * Unless required by applicable law or agreed to in writing,
  14  * software distributed under the License is distributed on an
  15  * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
  16  * KIND, either express or implied.  See the License for the
  17  * specific language governing permissions and limitations
  18  * under the License.
  19  */
  20
  21 import org.apache.archiva.checksum.ChecksumAlgorithm;
  22 import org.apache.archiva.checksum.ChecksummedFile;
  23 import org.apache.archiva.common.Try;
  24 import org.apache.archiva.common.utils.VersionUtil;
  25 import org.apache.archiva.filter.Filter;
  26 import org.apache.archiva.metadata.maven.MavenMetadataReader;
  27 import org.apache.archiva.metadata.model.ArtifactMetadata;
  28 import org.apache.archiva.metadata.model.ProjectMetadata;
  29 import org.apache.archiva.metadata.model.ProjectVersionMetadata;
  30 import org.apache.archiva.metadata.model.facets.RepositoryProblemFacet;
  31 import org.apache.archiva.metadata.repository.storage.*;
  32 import org.apache.archiva.model.ArchivaRepositoryMetadata;
  33 import org.apache.archiva.model.SnapshotVersion;
  34 import org.apache.archiva.policies.ProxyDownloadException;
  35 import org.apache.archiva.proxy.ProxyRegistry;
  36 import org.apache.archiva.proxy.maven.WagonFactory;
  37 import org.apache.archiva.proxy.model.NetworkProxy;
  38 import org.apache.archiva.proxy.model.ProxyConnector;
  39 import org.apache.archiva.proxy.model.RepositoryProxyHandler;
  40 import org.apache.archiva.repository.*;
  41 import org.apache.archiva.repository.content.Artifact;
  42 import org.apache.archiva.repository.content.ContentItem;
  43 import org.apache.archiva.repository.content.ItemSelector;
  44 import org.apache.archiva.repository.content.PathParser;
  45 import org.apache.archiva.repository.content.base.ArchivaItemSelector;
  46 import org.apache.archiva.repository.maven.MavenSystemManager;
  47 import org.apache.archiva.repository.metadata.RepositoryMetadataException;
  48 import org.apache.archiva.repository.storage.StorageAsset;
  49 import org.apache.commons.lang3.ArrayUtils;
  50 import org.apache.commons.lang3.StringUtils;
  51 import org.apache.maven.model.*;
  52 import org.apache.maven.model.building.*;
  53 import org.apache.maven.model.io.xpp3.MavenXpp3Reader;
  54 import org.codehaus.plexus.util.xml.pull.XmlPullParserException;
  55 import org.slf4j.Logger;
  56 import org.slf4j.LoggerFactory;
  57 import org.springframework.context.ApplicationContext;
  58 import org.springframework.stereotype.Service;
  59
  60 import javax.annotation.PostConstruct;
  61 import javax.inject.Inject;
  62 import javax.inject.Named;
  63 import java.io.FileNotFoundException;
  64 import java.io.IOException;
  65 import java.io.Reader;
  66 import java.nio.channels.Channels;
  67 import java.nio.charset.Charset;
  68 import java.nio.file.NoSuchFileException;
  69 import java.time.ZoneId;
  70 import java.time.ZonedDateTime;
  71 import java.util.*;
  72 import java.util.function.Predicate;
  73 import java.util.stream.Collectors;
  74
  75 // import java.io.FileNotFoundException;
  76
  77 /**
  78  * <p>
  79  * Maven 2 repository format storage implementation. This class currently takes parameters to indicate the repository to
  80  * deal with rather than being instantiated per-repository.
  81  * FIXME: instantiate one per repository and allocate permanently from a factory (which can be obtained within the session).
  82  * </p>
  83  * <p>
  84  * The session is passed in as an argument to obtain any necessary resources, rather than the class being instantiated
  85  * within the session in the context of a single managed repository's resolution needs.
  86  * </p>
  87  */
  88 @Service("repositoryStorage#maven2")
  89 public class Maven2RepositoryStorage
  90         implements RepositoryStorage {
  91
  92     private static final Logger log = LoggerFactory.getLogger(Maven2RepositoryStorage.class);
  93
  94     private ModelBuilder builder;
  95
  96     @Inject
  97     RepositoryRegistry repositoryRegistry;
  98
  99     @Inject
 100     @Named( "metadataReader#maven" )
 101     MavenMetadataReader metadataReader;
 102
 103     @Inject
 104     @Named("repositoryPathTranslator#maven2")
 105     private RepositoryPathTranslator pathTranslator;
 106
 107     @Inject
 108     private WagonFactory wagonFactory;
 109
 110     @Inject
 111     private ApplicationContext applicationContext;
 112
 113     @Inject
 114     @Named("pathParser#default")
 115     private PathParser pathParser;
 116
 117     @Inject
 118     private ProxyRegistry proxyRegistry;
 119
 120     @Inject
 121     private MavenSystemManager mavenSystemManager;
 122
 123     private static final String METADATA_FILENAME_START = "maven-metadata";
 124
 125     private static final String METADATA_FILENAME = METADATA_FILENAME_START + ".xml";
 126
 127     // This array must be lexically sorted
 128     private static final String[] IGNORED_FILES = {METADATA_FILENAME, "resolver-status.properties"};
 129
 130     private static final MavenXpp3Reader MAVEN_XPP_3_READER = new MavenXpp3Reader();
 131
 132
 133     @PostConstruct
 134     public void initialize() {
 135         builder = new DefaultModelBuilderFactory().newInstance();
 136
 137     }
 138
 139     @Override
 140     public ProjectMetadata readProjectMetadata(String repoId, String namespace, String projectId) {
 141         // TODO: could natively implement the "shared model" concept from the browse action to avoid needing it there?
 142         return null;
 143     }
 144
 145     @Override
 146     public ProjectVersionMetadata readProjectVersionMetadata(ReadMetadataRequest readMetadataRequest)
 147             throws RepositoryStorageMetadataNotFoundException, RepositoryStorageMetadataInvalidException,
 148             RepositoryStorageRuntimeException {
 149
 150         ManagedRepository managedRepository = repositoryRegistry.getManagedRepository(readMetadataRequest.getRepositoryId());
 151         boolean isReleases = managedRepository.getActiveReleaseSchemes().contains(ReleaseScheme.RELEASE);
 152         boolean isSnapshots = managedRepository.getActiveReleaseSchemes().contains(ReleaseScheme.SNAPSHOT);
 153         String artifactVersion = readMetadataRequest.getProjectVersion();
 154         // olamy: in case of browsing via the ui we can mix repos (parent of a SNAPSHOT can come from release repo)
 155         if (!readMetadataRequest.isBrowsingRequest()) {
 156             if (VersionUtil.isSnapshot(artifactVersion)) {
 157                 // skygo trying to improve speed by honoring managed configuration MRM-1658
 158                 if (isReleases && !isSnapshots) {
 159                     throw new RepositoryStorageRuntimeException("lookforsnaponreleaseonly",
 160                             "managed repo is configured for release only");
 161                 }
 162             } else {
 163                 if (!isReleases && isSnapshots) {
 164                     throw new RepositoryStorageRuntimeException("lookforsreleaseonsneponly",
 165                             "managed repo is configured for snapshot only");
 166                 }
 167             }
 168         }
 169         StorageAsset basedir = managedRepository.getRoot();
 170         if (VersionUtil.isSnapshot(artifactVersion)) {
 171             StorageAsset metadataFile = pathTranslator.toFile(basedir, readMetadataRequest.getNamespace(),
 172                     readMetadataRequest.getProjectId(), artifactVersion,
 173                     METADATA_FILENAME);
 174             try {
 175                 ArchivaRepositoryMetadata metadata = metadataReader.read(metadataFile);
 176
 177                 // re-adjust to timestamp if present, otherwise retain the original -SNAPSHOT filename
 178                 SnapshotVersion snapshotVersion = metadata.getSnapshotVersion();
 179                 if (snapshotVersion != null) {
 180                     artifactVersion =
 181                             artifactVersion.substring(0, artifactVersion.length() - 8); // remove SNAPSHOT from end
 182                     artifactVersion =
 183                             artifactVersion + snapshotVersion.getTimestamp() + "-" + snapshotVersion.getBuildNumber();
 184                 }
 185             } catch ( RepositoryMetadataException e) {
 186                 // unable to parse metadata - LOGGER it, and continue with the version as the original SNAPSHOT version
 187                 log.warn("Invalid metadata: {} - {}", metadataFile, e.getMessage());
 188             }
 189         }
 190
 191         // TODO: won't work well with some other layouts, might need to convert artifact parts to ID by path translator
 192         String id = readMetadataRequest.getProjectId() + "-" + artifactVersion + ".pom";
 193         StorageAsset file =
 194                 pathTranslator.toFile(basedir, readMetadataRequest.getNamespace(), readMetadataRequest.getProjectId(),
 195                         readMetadataRequest.getProjectVersion(), id);
 196
 197         if (!file.exists()) {
 198             // metadata could not be resolved
 199             throw new RepositoryStorageMetadataNotFoundException(
 200                     "The artifact's POM file '" + file.getPath() + "' was missing");
 201         }
 202
 203         // TODO: this is a workaround until we can properly resolve using proxies as well - this doesn't cache
 204         //       anything locally!
 205         List<RemoteRepository> remoteRepositories = new ArrayList<>();
 206         Map<String, NetworkProxy> networkProxies = new HashMap<>();
 207
 208         Map<String, List<ProxyConnector>> proxyConnectorsMap = proxyRegistry.getProxyConnectorAsMap();
 209         List<ProxyConnector> proxyConnectors = proxyConnectorsMap.get(readMetadataRequest.getRepositoryId());
 210         if (proxyConnectors != null) {
 211             for (ProxyConnector proxyConnector : proxyConnectors) {
 212                 RemoteRepository remoteRepoConfig =
 213                         repositoryRegistry.getRemoteRepository(proxyConnector.getTargetRepository().getId());
 214
 215                 if (remoteRepoConfig != null) {
 216                     remoteRepositories.add(remoteRepoConfig);
 217
 218                     NetworkProxy networkProxyConfig =
 219                             proxyRegistry.getNetworkProxy(proxyConnector.getProxyId());
 220
 221                     if (networkProxyConfig != null) {
 222                         // key/value: remote repo ID/proxy info
 223                         networkProxies.put(proxyConnector.getTargetRepository().getId(), networkProxyConfig);
 224                     }
 225                 }
 226             }
 227         }
 228
 229         // That's a browsing request so we can a mix of SNAPSHOT and release artifacts (especially with snapshots which
 230         // can have released parent pom
 231         if (readMetadataRequest.isBrowsingRequest()) {
 232             remoteRepositories.addAll(repositoryRegistry.getRemoteRepositories());
 233         }
 234
 235         ModelBuildingRequest req =
 236                 new DefaultModelBuildingRequest().setProcessPlugins(false).setPomFile(file.getFilePath().toFile()).setTwoPhaseBuilding(
 237                         false).setValidationLevel(ModelBuildingRequest.VALIDATION_LEVEL_MINIMAL);
 238
 239         //MRM-1607. olamy this will resolve jdk profiles on the current running archiva jvm
 240         req.setSystemProperties(System.getProperties());
 241
 242         // MRM-1411
 243         req.setModelResolver(
 244                 new RepositoryModelResolver(managedRepository, pathTranslator, wagonFactory, remoteRepositories,
 245                         networkProxies, managedRepository, mavenSystemManager, metadataReader));
 246
 247         Model model;
 248         try {
 249             model = builder.build(req).getEffectiveModel();
 250         } catch (ModelBuildingException e) {
 251             String msg = "The artifact's POM file '" + file + "' was invalid: " + e.getMessage();
 252
 253             List<ModelProblem> modelProblems = e.getProblems();
 254             for (ModelProblem problem : modelProblems) {
 255                 // MRM-1411, related to MRM-1335
 256                 // this means that the problem was that the parent wasn't resolved!
 257                 // olamy really hackhish but fail with java profile so use error message
 258                 // || ( StringUtils.startsWith( problem.getMessage(), "Failed to determine Java version for profile" ) )
 259                 // but setTwoPhaseBuilding(true) fix that
 260                 if (((problem.getException() instanceof FileNotFoundException
 261                         || problem.getException() instanceof NoSuchFileException
 262                 ) && e.getModelId() != null &&
 263                         !e.getModelId().equals(problem.getModelId()))) {
 264                     log.warn("The artifact's parent POM file '{}' cannot be resolved. "
 265                             + "Using defaults for project version metadata..", file);
 266
 267                     ProjectVersionMetadata metadata = new ProjectVersionMetadata();
 268                     metadata.setId(readMetadataRequest.getProjectVersion());
 269
 270                     MavenProjectFacet facet = new MavenProjectFacet();
 271                     facet.setGroupId(readMetadataRequest.getNamespace());
 272                     facet.setArtifactId(readMetadataRequest.getProjectId());
 273                     facet.setPackaging("jar");
 274                     metadata.addFacet(facet);
 275
 276                     String errMsg =
 277                             "Error in resolving artifact's parent POM file. " + (problem.getException() == null
 278                                     ? problem.getMessage()
 279                                     : problem.getException().getMessage());
 280                     RepositoryProblemFacet repoProblemFacet = new RepositoryProblemFacet();
 281                     repoProblemFacet.setRepositoryId(readMetadataRequest.getRepositoryId());
 282                     repoProblemFacet.setId(readMetadataRequest.getRepositoryId());
 283                     repoProblemFacet.setMessage(errMsg);
 284                     repoProblemFacet.setProblem(errMsg);
 285                     repoProblemFacet.setProject(readMetadataRequest.getProjectId());
 286                     repoProblemFacet.setVersion(readMetadataRequest.getProjectVersion());
 287                     repoProblemFacet.setNamespace(readMetadataRequest.getNamespace());
 288
 289                     metadata.addFacet(repoProblemFacet);
 290
 291                     return metadata;
 292                 }
 293             }
 294
 295             throw new RepositoryStorageMetadataInvalidException("invalid-pom", msg, e);
 296         }
 297
 298         // Check if the POM is in the correct location
 299         boolean correctGroupId = readMetadataRequest.getNamespace().equals(model.getGroupId());
 300         boolean correctArtifactId = readMetadataRequest.getProjectId().equals(model.getArtifactId());
 301         boolean correctVersion = readMetadataRequest.getProjectVersion().equals(model.getVersion());
 302         if (!correctGroupId || !correctArtifactId || !correctVersion) {
 303             StringBuilder message = new StringBuilder("Incorrect POM coordinates in '" + file + "':");
 304             if (!correctGroupId) {
 305                 message.append("\nIncorrect group ID: ").append(model.getGroupId());
 306             }
 307             if (!correctArtifactId) {
 308                 message.append("\nIncorrect artifact ID: ").append(model.getArtifactId());
 309             }
 310             if (!correctVersion) {
 311                 message.append("\nIncorrect version: ").append(model.getVersion());
 312             }
 313
 314             throw new RepositoryStorageMetadataInvalidException("mislocated-pom", message.toString());
 315         }
 316
 317         ProjectVersionMetadata metadata = new ProjectVersionMetadata();
 318         metadata.setCiManagement(convertCiManagement(model.getCiManagement()));
 319         metadata.setDescription(model.getDescription());
 320         metadata.setId(readMetadataRequest.getProjectVersion());
 321         metadata.setIssueManagement(convertIssueManagement(model.getIssueManagement()));
 322         metadata.setLicenses(convertLicenses(model.getLicenses()));
 323         metadata.setMailingLists(convertMailingLists(model.getMailingLists()));
 324         metadata.setDependencies(convertDependencies(model.getDependencies()));
 325         metadata.setName(model.getName());
 326         metadata.setOrganization(convertOrganization(model.getOrganization()));
 327         metadata.setScm(convertScm(model.getScm()));
 328         metadata.setUrl(model.getUrl());
 329         metadata.setProperties(model.getProperties());
 330
 331         MavenProjectFacet facet = new MavenProjectFacet();
 332         facet.setGroupId(model.getGroupId() != null ? model.getGroupId() : model.getParent().getGroupId());
 333         facet.setArtifactId(model.getArtifactId());
 334         facet.setPackaging(model.getPackaging());
 335         if (model.getParent() != null) {
 336             MavenProjectParent parent = new MavenProjectParent();
 337             parent.setGroupId(model.getParent().getGroupId());
 338             parent.setArtifactId(model.getParent().getArtifactId());
 339             parent.setVersion(model.getParent().getVersion());
 340             facet.setParent(parent);
 341         }
 342         metadata.addFacet(facet);
 343
 344         return metadata;
 345
 346
 347     }
 348
 349     public void setWagonFactory(WagonFactory wagonFactory) {
 350         this.wagonFactory = wagonFactory;
 351     }
 352
 353     private List<org.apache.archiva.metadata.model.Dependency> convertDependencies(List<Dependency> dependencies) {
 354         List<org.apache.archiva.metadata.model.Dependency> l = new ArrayList<>();
 355         for (Dependency dependency : dependencies) {
 356             org.apache.archiva.metadata.model.Dependency newDependency =
 357                     new org.apache.archiva.metadata.model.Dependency();
 358             newDependency.setArtifactId(dependency.getArtifactId());
 359             newDependency.setClassifier(dependency.getClassifier());
 360             newDependency.setNamespace(dependency.getGroupId());
 361             newDependency.setOptional(dependency.isOptional());
 362             newDependency.setScope(dependency.getScope());
 363             newDependency.setSystemPath(dependency.getSystemPath());
 364             newDependency.setType(dependency.getType());
 365             newDependency.setVersion(dependency.getVersion());
 366             l.add(newDependency);
 367         }
 368         return l;
 369     }
 370
 371     private org.apache.archiva.metadata.model.Scm convertScm(Scm scm) {
 372         org.apache.archiva.metadata.model.Scm newScm = null;
 373         if (scm != null) {
 374             newScm = new org.apache.archiva.metadata.model.Scm();
 375             newScm.setConnection(scm.getConnection());
 376             newScm.setDeveloperConnection(scm.getDeveloperConnection());
 377             newScm.setUrl(scm.getUrl());
 378         }
 379         return newScm;
 380     }
 381
 382     private org.apache.archiva.metadata.model.Organization convertOrganization(Organization organization) {
 383         org.apache.archiva.metadata.model.Organization org = null;
 384         if (organization != null) {
 385             org = new org.apache.archiva.metadata.model.Organization();
 386             org.setName(organization.getName());
 387             org.setUrl(organization.getUrl());
 388         }
 389         return org;
 390     }
 391
 392     private List<org.apache.archiva.metadata.model.License> convertLicenses(List<License> licenses) {
 393         List<org.apache.archiva.metadata.model.License> l = new ArrayList<>();
 394         for (License license : licenses) {
 395             org.apache.archiva.metadata.model.License newLicense = new org.apache.archiva.metadata.model.License();
 396             newLicense.setName(license.getName());
 397             newLicense.setUrl(license.getUrl());
 398             l.add(newLicense);
 399         }
 400         return l;
 401     }
 402
 403     private List<org.apache.archiva.metadata.model.MailingList> convertMailingLists(List<MailingList> mailingLists) {
 404         List<org.apache.archiva.metadata.model.MailingList> l = new ArrayList<>();
 405         for (MailingList mailingList : mailingLists) {
 406             org.apache.archiva.metadata.model.MailingList newMailingList =
 407                     new org.apache.archiva.metadata.model.MailingList();
 408             newMailingList.setName(mailingList.getName());
 409             newMailingList.setMainArchiveUrl(mailingList.getArchive());
 410             newMailingList.setPostAddress(mailingList.getPost());
 411             newMailingList.setSubscribeAddress(mailingList.getSubscribe());
 412             newMailingList.setUnsubscribeAddress(mailingList.getUnsubscribe());
 413             newMailingList.setOtherArchives(mailingList.getOtherArchives());
 414             l.add(newMailingList);
 415         }
 416         return l;
 417     }
 418
 419     private org.apache.archiva.metadata.model.IssueManagement convertIssueManagement(IssueManagement issueManagement) {
 420         org.apache.archiva.metadata.model.IssueManagement im = null;
 421         if (issueManagement != null) {
 422             im = new org.apache.archiva.metadata.model.IssueManagement();
 423             im.setSystem(issueManagement.getSystem());
 424             im.setUrl(issueManagement.getUrl());
 425         }
 426         return im;
 427     }
 428
 429     private org.apache.archiva.metadata.model.CiManagement convertCiManagement(CiManagement ciManagement) {
 430         org.apache.archiva.metadata.model.CiManagement ci = null;
 431         if (ciManagement != null) {
 432             ci = new org.apache.archiva.metadata.model.CiManagement();
 433             ci.setSystem(ciManagement.getSystem());
 434             ci.setUrl(ciManagement.getUrl());
 435         }
 436         return ci;
 437     }
 438
 439     @Override
 440     public Collection<String> listRootNamespaces(String repoId, Filter<String> filter)
 441             throws RepositoryStorageRuntimeException {
 442         StorageAsset dir = getRepositoryBasedir(repoId);
 443
 444         return getSortedFiles(dir, filter);
 445     }
 446
 447     private static Collection<String> getSortedFiles(StorageAsset dir, Filter<String> filter) {
 448
 449         final Predicate<StorageAsset> dFilter = new DirectoryFilter(filter);
 450         return dir.list().stream().filter(f -> f.isContainer())
 451                 .filter(dFilter)
 452                 .map(path -> path.getName().toString())
 453                 .sorted().collect(Collectors.toList());
 454
 455     }
 456
 457     private StorageAsset getRepositoryBasedir(String repoId)
 458             throws RepositoryStorageRuntimeException {
 459         ManagedRepository repository = repositoryRegistry.getManagedRepository(repoId);
 460
 461         return repository.getRoot();
 462     }
 463
 464     @Override
 465     public Collection<String> listNamespaces(String repoId, String namespace, Filter<String> filter)
 466             throws RepositoryStorageRuntimeException {
 467         StorageAsset dir = pathTranslator.toFile(getRepositoryBasedir(repoId), namespace);
 468         if (!(dir.exists()) && !dir.isContainer()) {
 469             return Collections.emptyList();
 470         }
 471         // scan all the directories which are potential namespaces. Any directories known to be projects are excluded
 472         Predicate<StorageAsset> dFilter = new DirectoryFilter(filter);
 473         return dir.list().stream().filter(dFilter).filter(path -> !isProject(path, filter)).map(path -> path.getName().toString())
 474                 .sorted().collect(Collectors.toList());
 475     }
 476
 477     @Override
 478     public Collection<String> listProjects(String repoId, String namespace, Filter<String> filter)
 479             throws RepositoryStorageRuntimeException {
 480         StorageAsset dir = pathTranslator.toFile(getRepositoryBasedir(repoId), namespace);
 481         if (!(dir.exists() && dir.isContainer())) {
 482             return Collections.emptyList();
 483         }
 484         // scan all directories in the namespace, and only include those that are known to be projects
 485         final Predicate<StorageAsset> dFilter = new DirectoryFilter(filter);
 486         return dir.list().stream().filter(dFilter).filter(path -> isProject(path, filter)).map(path -> path.getName().toString())
 487                 .sorted().collect(Collectors.toList());
 488
 489     }
 490
 491     @Override
 492     public Collection<String> listProjectVersions(String repoId, String namespace, String projectId,
 493                                                   Filter<String> filter)
 494             throws RepositoryStorageRuntimeException {
 495         StorageAsset dir = pathTranslator.toFile(getRepositoryBasedir(repoId), namespace, projectId);
 496         if (!(dir.exists() && dir.isContainer())) {
 497             return Collections.emptyList();
 498         }
 499
 500         // all directories in a project directory can be considered a version
 501         return getSortedFiles(dir, filter);
 502     }
 503
 504     @Override
 505     public Collection<ArtifactMetadata> readArtifactsMetadata(ReadMetadataRequest readMetadataRequest)
 506             throws RepositoryStorageRuntimeException {
 507         StorageAsset dir = pathTranslator.toFile(getRepositoryBasedir(readMetadataRequest.getRepositoryId()),
 508                 readMetadataRequest.getNamespace(), readMetadataRequest.getProjectId(),
 509                 readMetadataRequest.getProjectVersion());
 510         if (!(dir.exists() && dir.isContainer())) {
 511             return Collections.emptyList();
 512         }
 513
 514         // all files that are not metadata and not a checksum / signature are considered artifacts
 515         final Predicate<StorageAsset> dFilter = new ArtifactDirectoryFilter(readMetadataRequest.getFilter());
 516         // Returns a map TRUE -> (success values), FALSE -> (Exceptions)
 517         Map<Boolean, List<Try<ArtifactMetadata>>> result = dir.list().stream().filter(dFilter).map(path -> {
 518                     try {
 519                         return Try.success(getArtifactFromFile(readMetadataRequest.getRepositoryId(), readMetadataRequest.getNamespace(),
 520                                 readMetadataRequest.getProjectId(), readMetadataRequest.getProjectVersion(),
 521                                 path));
 522                     } catch (Exception e) {
 523                         log.debug("Could not create metadata for {}:  {}", path, e.getMessage(), e);
 524                         return Try.<ArtifactMetadata>failure(e);
 525                     }
 526                 }
 527         ).collect(Collectors.groupingBy(Try::isSuccess));
 528         if (result.containsKey(Boolean.FALSE) && result.get(Boolean.FALSE).size() > 0 && (!result.containsKey(Boolean.TRUE) || result.get(Boolean.TRUE).size() == 0)) {
 529             log.error("Could not get artifact metadata. Directory: {}. Number of errors {}.", dir, result.get(Boolean.FALSE).size());
 530             Try<ArtifactMetadata> failure = result.get(Boolean.FALSE).get(0);
 531             log.error("Sample exception {}", failure.getError().getMessage(), failure.getError());
 532             throw new RepositoryStorageRuntimeException(readMetadataRequest.getRepositoryId(), "Could not retrieve metadata of the files");
 533         } else {
 534             if (!result.containsKey(Boolean.TRUE) || result.get(Boolean.TRUE) == null) {
 535                 return Collections.emptyList();
 536             }
 537             return result.get(Boolean.TRUE).stream().map(tr -> tr.get()).collect(Collectors.toList());
 538         }
 539
 540     }
 541
 542     @Override
 543     public ArtifactMetadata readArtifactMetadataFromPath(String repoId, String path)
 544             throws RepositoryStorageRuntimeException {
 545         ArtifactMetadata metadata = pathTranslator.getArtifactForPath(repoId, path);
 546
 547         try {
 548             populateArtifactMetadataFromFile(metadata, getRepositoryBasedir(repoId).resolve(path));
 549         } catch (IOException e) {
 550             throw new RepositoryStorageRuntimeException(repoId, "Error during metadata retrieval of " + path + " :" + e.getMessage(), e);
 551         }
 552
 553         return metadata;
 554     }
 555
 556     private ArtifactMetadata getArtifactFromFile(String repoId, String namespace, String projectId,
 557                                                  String projectVersion, StorageAsset file) throws IOException {
 558         ArtifactMetadata metadata =
 559                 pathTranslator.getArtifactFromId(repoId, namespace, projectId, projectVersion, file.getName());
 560
 561         populateArtifactMetadataFromFile(metadata, file);
 562
 563         return metadata;
 564     }
 565
 566     @Override
 567     public ItemSelector applyServerSideRelocation(ManagedRepository managedRepository, ItemSelector artifactSelector)
 568         throws ProxyDownloadException {
 569         if ("pom".equals(artifactSelector.getType())) {
 570             return artifactSelector;
 571         }
 572
 573         // Build the artifact POM reference
 574         BaseRepositoryContentLayout layout;
 575         try
 576         {
 577             layout = managedRepository.getContent( ).getLayout( BaseRepositoryContentLayout.class );
 578         }
 579         catch ( LayoutException e )
 580         {
 581             throw new ProxyDownloadException( "Could not set layout " + e.getMessage( ), new HashMap<>(  ) );
 582         }
 583
 584         RepositoryType repositoryType = managedRepository.getType();
 585         if (!proxyRegistry.hasHandler(repositoryType)) {
 586             throw new ProxyDownloadException("No proxy handler found for repository type " + repositoryType, new HashMap<>());
 587         }
 588
 589
 590
 591         ItemSelector selector = ArchivaItemSelector.builder( )
 592             .withNamespace( artifactSelector.getNamespace( ) )
 593             .withProjectId( artifactSelector.getArtifactId( ) )
 594             .withArtifactId( artifactSelector.getArtifactId( ) )
 595             .withVersion( artifactSelector.getVersion( ) )
 596             .withArtifactVersion( artifactSelector.getVersion( ) )
 597             .withType( "pom" ).build( );
 598
 599         Artifact pom = layout.getArtifact( selector );
 600
 601         RepositoryProxyHandler proxyHandler = proxyRegistry.getHandler(repositoryType).get(0);
 602
 603         // Get the artifact POM from proxied repositories if needed
 604         proxyHandler.fetchFromProxies(managedRepository, pom);
 605
 606         // Open and read the POM from the managed repo
 607
 608         if (!pom.exists()) {
 609             return artifactSelector;
 610         }
 611
 612         try {
 613             // MavenXpp3Reader leaves the file open, so we need to close it ourselves.
 614
 615             Model model;
 616             try (Reader reader = Channels.newReader(pom.getAsset().getReadChannel(), Charset.defaultCharset().name())) {
 617                 model = MAVEN_XPP_3_READER.read(reader);
 618             }
 619
 620             DistributionManagement dist = model.getDistributionManagement();
 621             if (dist != null) {
 622                 Relocation relocation = dist.getRelocation();
 623                 if (relocation != null) {
 624                     ArchivaItemSelector.Builder relocatedBuilder = ArchivaItemSelector.builder( );
 625                     // artifact is relocated : update the repositoryPath
 626                     if (relocation.getGroupId() != null) {
 627                         relocatedBuilder.withNamespace( relocation.getGroupId( ) );
 628                     } else {
 629                         relocatedBuilder.withNamespace( artifactSelector.getNamespace( ) );
 630                     }
 631                     if (relocation.getArtifactId() != null) {
 632                         relocatedBuilder.withArtifactId( relocation.getArtifactId( ) );
 633                     } else {
 634                         relocatedBuilder.withArtifactId( artifactSelector.getArtifactId( ) );
 635                     }
 636                     if (relocation.getVersion() != null)
 637                     {
 638                         relocatedBuilder.withVersion( relocation.getVersion( ) );
 639                     } else {
 640                         relocatedBuilder.withVersion( artifactSelector.getVersion( ) );
 641                     }
 642                     return relocatedBuilder.withArtifactVersion( artifactSelector.getArtifactVersion( ) )
 643                         .withClassifier( artifactSelector.getClassifier( ) )
 644                         .withType( artifactSelector.getType( ) )
 645                         .withProjectId( artifactSelector.getProjectId( ) )
 646                         .withExtension( artifactSelector.getExtension( ) )
 647                         .build( );
 648                 }
 649             }
 650         } catch (IOException e) {
 651             // Unable to read POM : ignore.
 652         } catch (XmlPullParserException e) {
 653             // Invalid POM : ignore
 654         }
 655         return artifactSelector;
 656     }
 657
 658
 659     @Override
 660     public String getFilePath(String requestPath, org.apache.archiva.repository.ManagedRepository managedRepository) {
 661         // managedRepository can be null
 662         // extract artifact reference from url
 663         // groupId:artifactId:version:packaging:classifier
 664         //org/apache/archiva/archiva-checksum/1.4-M4-SNAPSHOT/archiva-checksum-1.4-M4-SNAPSHOT.jar
 665         String logicalResource = null;
 666         String requestPathInfo = StringUtils.defaultString(requestPath);
 667
 668         //remove prefix ie /repository/blah becomes /blah
 669         requestPathInfo = removePrefix(requestPathInfo);
 670
 671         // Remove prefixing slash as the repository id doesn't contain it;
 672         if (requestPathInfo.startsWith("/")) {
 673             requestPathInfo = requestPathInfo.substring(1);
 674         }
 675
 676         int slash = requestPathInfo.indexOf('/');
 677         if (slash > 0) {
 678             logicalResource = requestPathInfo.substring(slash);
 679
 680             if (logicalResource.endsWith("/..")) {
 681                 logicalResource += "/";
 682             }
 683
 684             if (logicalResource != null && logicalResource.startsWith("//")) {
 685                 logicalResource = logicalResource.substring(1);
 686             }
 687
 688             if (logicalResource == null) {
 689                 logicalResource = "/";
 690             }
 691         } else {
 692             logicalResource = "/";
 693         }
 694         return logicalResource;
 695
 696     }
 697
 698     @Override
 699     public String getFilePathWithVersion(final String requestPath, ManagedRepositoryContent managedRepositoryContent)
 700             throws RelocationException
 701     {
 702
 703         if (StringUtils.endsWith(requestPath, METADATA_FILENAME)) {
 704             return getFilePath(requestPath, managedRepositoryContent.getRepository());
 705         }
 706
 707         String filePath = getFilePath(requestPath, managedRepositoryContent.getRepository());
 708         Artifact artifact;
 709         try
 710         {
 711             ContentItem item = managedRepositoryContent.toItem( filePath );
 712             artifact = managedRepositoryContent.getLayout( BaseRepositoryContentLayout.class ).adaptItem( Artifact.class, item );
 713         }
 714         catch ( LayoutException e )
 715         {
 716             return filePath;
 717         }
 718
 719         if (VersionUtil.isGenericSnapshot(artifact.getArtifactVersion())) {
 720             // read maven metadata to get last timestamp
 721             StorageAsset metadataDir = managedRepositoryContent.getRepository().getAsset(filePath).getParent();
 722             if (!metadataDir.exists()) {
 723                 return filePath;
 724             }
 725             StorageAsset metadataFile = metadataDir.resolve(METADATA_FILENAME);
 726             if (!metadataFile.exists()) {
 727                 return filePath;
 728             }
 729             ArchivaRepositoryMetadata archivaRepositoryMetadata = null;
 730             try
 731             {
 732                 archivaRepositoryMetadata = metadataReader.read(metadataFile);
 733             }
 734             catch ( RepositoryMetadataException e )
 735             {
 736                 log.error( "Could not read metadata {}", e.getMessage( ), e );
 737                 return filePath;
 738             }
 739             int buildNumber = archivaRepositoryMetadata.getSnapshotVersion().getBuildNumber();
 740             String timestamp = archivaRepositoryMetadata.getSnapshotVersion().getTimestamp();
 741
 742             // MRM-1846
 743             if (buildNumber < 1 && timestamp == null) {
 744                 return filePath;
 745             }
 746
 747             // org/apache/archiva/archiva-checksum/1.4-M4-SNAPSHOT/archiva-checksum-1.4-M4-SNAPSHOT.jar
 748             // ->  archiva-checksum-1.4-M4-20130425.081822-1.jar
 749
 750             filePath = StringUtils.replace(filePath, //
 751                     artifact.getId() //
 752                             + "-" + artifact.getVersion().getId(), //
 753                     artifact.getId() //
 754                             + "-" + StringUtils.remove(artifact.getArtifactVersion(),
 755                             "-" + VersionUtil.SNAPSHOT) //
 756                             + "-" + timestamp //
 757                             + "-" + buildNumber);
 758
 759             throw new RelocationException("/repository/" + managedRepositoryContent.getRepository().getId() +
 760                     (StringUtils.startsWith(filePath, "/") ? "" : "/") + filePath,
 761                     RelocationException.RelocationType.TEMPORARY);
 762
 763         }
 764
 765         return filePath;
 766     }
 767
 768     //-----------------------------
 769     // internal
 770     //-----------------------------
 771
 772     /**
 773      * FIXME remove
 774      *
 775      * @param href
 776      * @return
 777      */
 778     private static String removePrefix(final String href) {
 779         String[] parts = StringUtils.split(href, '/');
 780         parts = (String[]) ArrayUtils.subarray(parts, 1, parts.length);
 781         if (parts == null || parts.length == 0) {
 782             return "/";
 783         }
 784
 785         String joinedString = StringUtils.join(parts, '/');
 786         if (href.endsWith("/")) {
 787             joinedString = joinedString + "/";
 788         }
 789
 790         return joinedString;
 791     }
 792
 793     private static void populateArtifactMetadataFromFile(ArtifactMetadata metadata, StorageAsset file) throws IOException {
 794         metadata.setWhenGathered(ZonedDateTime.now(ZoneId.of("GMT")));
 795         metadata.setFileLastModified(file.getModificationTime().toEpochMilli());
 796         ChecksummedFile checksummedFile = new ChecksummedFile(file.getFilePath());
 797         try {
 798             metadata.setMd5(checksummedFile.calculateChecksum(ChecksumAlgorithm.MD5));
 799         } catch (IOException e) {
 800             log.error("Unable to checksum file {}: {},MD5", file, e.getMessage());
 801         }
 802         try {
 803             metadata.setSha1(checksummedFile.calculateChecksum(ChecksumAlgorithm.SHA1));
 804         } catch (IOException e) {
 805             log.error("Unable to checksum file {}: {},SHA1", file, e.getMessage());
 806         }
 807         metadata.setSize(file.getSize());
 808     }
 809
 810     private boolean isProject(StorageAsset dir, Filter<String> filter) {
 811         // scan directories for a valid project version subdirectory, meaning this must be a project directory
 812         final Predicate<StorageAsset> dFilter = new DirectoryFilter(filter);
 813         boolean projFound = dir.list().stream().filter(dFilter)
 814                 .anyMatch(path -> isProjectVersion(path));
 815         if (projFound) {
 816             return true;
 817         }
 818
 819         // if a metadata file is present, check if this is the "artifactId" directory, marking it as a project
 820         ArchivaRepositoryMetadata metadata = readMetadata(dir);
 821         if (metadata != null && dir.getName().toString().equals(metadata.getArtifactId())) {
 822             return true;
 823         }
 824
 825         return false;
 826     }
 827
 828     private boolean isProjectVersion(StorageAsset dir) {
 829         final String artifactId = dir.getParent().getName();
 830         final String projectVersion = dir.getName();
 831
 832         // check if there is a POM artifact file to ensure it is a version directory
 833
 834         Predicate<StorageAsset> filter;
 835         if (VersionUtil.isSnapshot(projectVersion)) {
 836             filter = new PomFilenameFilter(artifactId, projectVersion);
 837         } else {
 838             final String pomFile = artifactId + "-" + projectVersion + ".pom";
 839             filter = new PomFileFilter(pomFile);
 840         }
 841         if (dir.list().stream().filter(f -> !f.isContainer()).anyMatch(filter)) {
 842             return true;
 843         }
 844         // if a metadata file is present, check if this is the "version" directory, marking it as a project version
 845         ArchivaRepositoryMetadata metadata = readMetadata(dir);
 846         if (metadata != null && projectVersion.equals(metadata.getVersion())) {
 847             return true;
 848         }
 849
 850         return false;
 851     }
 852
 853     private ArchivaRepositoryMetadata readMetadata(StorageAsset directory) {
 854         ArchivaRepositoryMetadata metadata = null;
 855         StorageAsset metadataFile = directory.resolve(METADATA_FILENAME);
 856         if (metadataFile.exists()) {
 857             try {
 858                 metadata = metadataReader.read(metadataFile);
 859             } catch ( RepositoryMetadataException e )
 860             {
 861                 // Ignore missing or invalid metadata
 862             }
 863         }
 864         return metadata;
 865     }
 866
 867     private static class DirectoryFilter
 868             implements Predicate<StorageAsset> {
 869         private final Filter<String> filter;
 870
 871         public DirectoryFilter(Filter<String> filter) {
 872             this.filter = filter;
 873         }
 874
 875         @Override
 876         public boolean test(StorageAsset dir) {
 877             final String name = dir.getName();
 878             if (!filter.accept(name)) {
 879                 return false;
 880             } else if (name.startsWith(".")) {
 881                 return false;
 882             } else if (!dir.isContainer()) {
 883                 return false;
 884             }
 885             return true;
 886         }
 887     }
 888
 889     private static class ArtifactDirectoryFilter
 890             implements Predicate<StorageAsset> {
 891         private final Filter<String> filter;
 892
 893         private ArtifactDirectoryFilter(Filter<String> filter) {
 894             this.filter = filter;
 895         }
 896
 897         @Override
 898         public boolean test(StorageAsset file) {
 899             final Set<String> checksumExts = ChecksumAlgorithm.getAllExtensions();
 900             final String path = file.getPath();
 901             final String name = file.getName();
 902             final String extension = StringUtils.substringAfterLast(name, ".").toLowerCase();
 903             // TODO compare to logic in maven-repository-layer
 904             if (file.isContainer()) {
 905                 return false;
 906             } else if (!filter.accept(name)) {
 907                 return false;
 908             } else if (name.startsWith(".") || path.contains("/.") ) {
 909                 return false;
 910             } else if (checksumExts.contains(extension)) {
 911                 return false;
 912             } else if (Arrays.binarySearch(IGNORED_FILES, name) >= 0) {
 913                 return false;
 914             }
 915             // some files from remote repositories can have name like maven-metadata-archiva-vm-all-public.xml
 916             else if (StringUtils.startsWith(name, METADATA_FILENAME_START) && StringUtils.endsWith(name, ".xml")) {
 917                 return false;
 918             }
 919
 920             return true;
 921
 922         }
 923     }
 924
 925
 926     private static final class PomFilenameFilter
 927             implements Predicate<StorageAsset> {
 928
 929         private final String artifactId, projectVersion;
 930
 931         private PomFilenameFilter(String artifactId, String projectVersion) {
 932             this.artifactId = artifactId;
 933             this.projectVersion = projectVersion;
 934         }
 935
 936         @Override
 937         public boolean test(StorageAsset dir) {
 938             final String name = dir.getName();
 939             if (name.startsWith(artifactId + "-") && name.endsWith(".pom")) {
 940                 String v = name.substring(artifactId.length() + 1, name.length() - 4);
 941                 v = VersionUtil.getBaseVersion(v);
 942                 if (v.equals(projectVersion)) {
 943                     return true;
 944                 }
 945             }
 946             return false;
 947         }
 948
 949     }
 950
 951     private static class PomFileFilter
 952             implements Predicate<StorageAsset> {
 953         private final String pomFile;
 954
 955         private PomFileFilter(String pomFile) {
 956             this.pomFile = pomFile;
 957         }
 958
 959         @Override
 960         public boolean test(StorageAsset dir) {
 961             return pomFile.equals(dir.getName());
 962         }
 963     }
 964
 965
 966     public PathParser getPathParser() {
 967         return pathParser;
 968     }
 969
 970     public void setPathParser(PathParser pathParser) {
 971         this.pathParser = pathParser;
 972     }
 973 }