source.dussan.org Git - archiva.git/blob

   1 package org.apache.archiva.repository.maven.metadata.storage;
   2
   3 /*
   4  * Licensed to the Apache Software Foundation (ASF) under one
   5  * or more contributor license agreements.  See the NOTICE file
   6  * distributed with this work for additional information
   7  * regarding copyright ownership.  The ASF licenses this file
   8  * to you under the Apache License, Version 2.0 (the
   9  * "License"); you may not use this file except in compliance
  10  * with the License.  You may obtain a copy of the License at
  11  *
  12  * http://www.apache.org/licenses/LICENSE-2.0
  13  * Unless required by applicable law or agreed to in writing,
  14  * software distributed under the License is distributed on an
  15  * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
  16  * KIND, either express or implied.  See the License for the
  17  * specific language governing permissions and limitations
  18  * under the License.
  19  */
  20
  21 import org.apache.archiva.checksum.ChecksumAlgorithm;
  22 import org.apache.archiva.checksum.ChecksummedFile;
  23 import org.apache.archiva.common.Try;
  24 import org.apache.archiva.common.utils.VersionUtil;
  25 import org.apache.archiva.filter.Filter;
  26 import org.apache.archiva.metadata.maven.MavenMetadataReader;
  27 import org.apache.archiva.metadata.model.ArtifactMetadata;
  28 import org.apache.archiva.metadata.model.ProjectMetadata;
  29 import org.apache.archiva.metadata.model.ProjectVersionMetadata;
  30 import org.apache.archiva.metadata.model.facets.RepositoryProblemFacet;
  31 import org.apache.archiva.metadata.repository.storage.*;
  32 import org.apache.archiva.model.ArchivaRepositoryMetadata;
  33 import org.apache.archiva.model.ArtifactReference;
  34 import org.apache.archiva.model.SnapshotVersion;
  35 import org.apache.archiva.policies.ProxyDownloadException;
  36 import org.apache.archiva.proxy.ProxyRegistry;
  37 import org.apache.archiva.proxy.maven.WagonFactory;
  38 import org.apache.archiva.proxy.model.NetworkProxy;
  39 import org.apache.archiva.proxy.model.ProxyConnector;
  40 import org.apache.archiva.proxy.model.RepositoryProxyHandler;
  41 import org.apache.archiva.repository.*;
  42 import org.apache.archiva.repository.content.Artifact;
  43 import org.apache.archiva.repository.content.ItemSelector;
  44 import org.apache.archiva.repository.content.PathParser;
  45 import org.apache.archiva.repository.content.base.ArchivaItemSelector;
  46 import org.apache.archiva.repository.maven.MavenSystemManager;
  47 import org.apache.archiva.repository.metadata.RepositoryMetadataException;
  48 import org.apache.archiva.repository.storage.StorageAsset;
  49 import org.apache.commons.lang3.ArrayUtils;
  50 import org.apache.commons.lang3.StringUtils;
  51 import org.apache.maven.model.*;
  52 import org.apache.maven.model.building.*;
  53 import org.apache.maven.model.io.xpp3.MavenXpp3Reader;
  54 import org.codehaus.plexus.util.xml.pull.XmlPullParserException;
  55 import org.slf4j.Logger;
  56 import org.slf4j.LoggerFactory;
  57 import org.springframework.context.ApplicationContext;
  58 import org.springframework.stereotype.Service;
  59
  60 import javax.annotation.PostConstruct;
  61 import javax.inject.Inject;
  62 import javax.inject.Named;
  63 import java.io.FileNotFoundException;
  64 import java.io.IOException;
  65 import java.io.Reader;
  66 import java.nio.channels.Channels;
  67 import java.nio.charset.Charset;
  68 import java.nio.file.NoSuchFileException;
  69 import java.time.ZoneId;
  70 import java.time.ZonedDateTime;
  71 import java.util.*;
  72 import java.util.function.Predicate;
  73 import java.util.stream.Collectors;
  74
  75 // import java.io.FileNotFoundException;
  76
  77 /**
  78  * <p>
  79  * Maven 2 repository format storage implementation. This class currently takes parameters to indicate the repository to
  80  * deal with rather than being instantiated per-repository.
  81  * FIXME: instantiate one per repository and allocate permanently from a factory (which can be obtained within the session).
  82  * </p>
  83  * <p>
  84  * The session is passed in as an argument to obtain any necessary resources, rather than the class being instantiated
  85  * within the session in the context of a single managed repository's resolution needs.
  86  * </p>
  87  */
  88 @Service("repositoryStorage#maven2")
  89 public class Maven2RepositoryStorage
  90         implements RepositoryStorage {
  91
  92     private static final Logger log = LoggerFactory.getLogger(Maven2RepositoryStorage.class);
  93
  94     private ModelBuilder builder;
  95
  96     @Inject
  97     RepositoryRegistry repositoryRegistry;
  98
  99     @Inject
 100     @Named( "metadataReader#maven" )
 101     MavenMetadataReader metadataReader;
 102
 103     @Inject
 104     @Named("repositoryPathTranslator#maven2")
 105     private RepositoryPathTranslator pathTranslator;
 106
 107     @Inject
 108     private WagonFactory wagonFactory;
 109
 110     @Inject
 111     private ApplicationContext applicationContext;
 112
 113     @Inject
 114     @Named("pathParser#default")
 115     private PathParser pathParser;
 116
 117     @Inject
 118     private ProxyRegistry proxyRegistry;
 119
 120     @Inject
 121     private MavenSystemManager mavenSystemManager;
 122
 123     private static final String METADATA_FILENAME_START = "maven-metadata";
 124
 125     private static final String METADATA_FILENAME = METADATA_FILENAME_START + ".xml";
 126
 127     // This array must be lexically sorted
 128     private static final String[] IGNORED_FILES = {METADATA_FILENAME, "resolver-status.properties"};
 129
 130     private static final MavenXpp3Reader MAVEN_XPP_3_READER = new MavenXpp3Reader();
 131
 132
 133     @PostConstruct
 134     public void initialize() {
 135         builder = new DefaultModelBuilderFactory().newInstance();
 136
 137     }
 138
 139     @Override
 140     public ProjectMetadata readProjectMetadata(String repoId, String namespace, String projectId) {
 141         // TODO: could natively implement the "shared model" concept from the browse action to avoid needing it there?
 142         return null;
 143     }
 144
 145     @Override
 146     public ProjectVersionMetadata readProjectVersionMetadata(ReadMetadataRequest readMetadataRequest)
 147             throws RepositoryStorageMetadataNotFoundException, RepositoryStorageMetadataInvalidException,
 148             RepositoryStorageRuntimeException {
 149
 150         ManagedRepository managedRepository = repositoryRegistry.getManagedRepository(readMetadataRequest.getRepositoryId());
 151         boolean isReleases = managedRepository.getActiveReleaseSchemes().contains(ReleaseScheme.RELEASE);
 152         boolean isSnapshots = managedRepository.getActiveReleaseSchemes().contains(ReleaseScheme.SNAPSHOT);
 153         String artifactVersion = readMetadataRequest.getProjectVersion();
 154         // olamy: in case of browsing via the ui we can mix repos (parent of a SNAPSHOT can come from release repo)
 155         if (!readMetadataRequest.isBrowsingRequest()) {
 156             if (VersionUtil.isSnapshot(artifactVersion)) {
 157                 // skygo trying to improve speed by honoring managed configuration MRM-1658
 158                 if (isReleases && !isSnapshots) {
 159                     throw new RepositoryStorageRuntimeException("lookforsnaponreleaseonly",
 160                             "managed repo is configured for release only");
 161                 }
 162             } else {
 163                 if (!isReleases && isSnapshots) {
 164                     throw new RepositoryStorageRuntimeException("lookforsreleaseonsneponly",
 165                             "managed repo is configured for snapshot only");
 166                 }
 167             }
 168         }
 169         StorageAsset basedir = managedRepository.getRoot();
 170         if (VersionUtil.isSnapshot(artifactVersion)) {
 171             StorageAsset metadataFile = pathTranslator.toFile(basedir, readMetadataRequest.getNamespace(),
 172                     readMetadataRequest.getProjectId(), artifactVersion,
 173                     METADATA_FILENAME);
 174             try {
 175                 ArchivaRepositoryMetadata metadata = metadataReader.read(metadataFile);
 176
 177                 // re-adjust to timestamp if present, otherwise retain the original -SNAPSHOT filename
 178                 SnapshotVersion snapshotVersion = metadata.getSnapshotVersion();
 179                 if (snapshotVersion != null) {
 180                     artifactVersion =
 181                             artifactVersion.substring(0, artifactVersion.length() - 8); // remove SNAPSHOT from end
 182                     artifactVersion =
 183                             artifactVersion + snapshotVersion.getTimestamp() + "-" + snapshotVersion.getBuildNumber();
 184                 }
 185             } catch ( RepositoryMetadataException e) {
 186                 // unable to parse metadata - LOGGER it, and continue with the version as the original SNAPSHOT version
 187                 log.warn("Invalid metadata: {} - {}", metadataFile, e.getMessage());
 188             }
 189         }
 190
 191         // TODO: won't work well with some other layouts, might need to convert artifact parts to ID by path translator
 192         String id = readMetadataRequest.getProjectId() + "-" + artifactVersion + ".pom";
 193         StorageAsset file =
 194                 pathTranslator.toFile(basedir, readMetadataRequest.getNamespace(), readMetadataRequest.getProjectId(),
 195                         readMetadataRequest.getProjectVersion(), id);
 196
 197         if (!file.exists()) {
 198             // metadata could not be resolved
 199             throw new RepositoryStorageMetadataNotFoundException(
 200                     "The artifact's POM file '" + file.getPath() + "' was missing");
 201         }
 202
 203         // TODO: this is a workaround until we can properly resolve using proxies as well - this doesn't cache
 204         //       anything locally!
 205         List<RemoteRepository> remoteRepositories = new ArrayList<>();
 206         Map<String, NetworkProxy> networkProxies = new HashMap<>();
 207
 208         Map<String, List<ProxyConnector>> proxyConnectorsMap = proxyRegistry.getProxyConnectorAsMap();
 209         List<ProxyConnector> proxyConnectors = proxyConnectorsMap.get(readMetadataRequest.getRepositoryId());
 210         if (proxyConnectors != null) {
 211             for (ProxyConnector proxyConnector : proxyConnectors) {
 212                 RemoteRepository remoteRepoConfig =
 213                         repositoryRegistry.getRemoteRepository(proxyConnector.getTargetRepository().getId());
 214
 215                 if (remoteRepoConfig != null) {
 216                     remoteRepositories.add(remoteRepoConfig);
 217
 218                     NetworkProxy networkProxyConfig =
 219                             proxyRegistry.getNetworkProxy(proxyConnector.getProxyId());
 220
 221                     if (networkProxyConfig != null) {
 222                         // key/value: remote repo ID/proxy info
 223                         networkProxies.put(proxyConnector.getTargetRepository().getId(), networkProxyConfig);
 224                     }
 225                 }
 226             }
 227         }
 228
 229         // That's a browsing request so we can a mix of SNAPSHOT and release artifacts (especially with snapshots which
 230         // can have released parent pom
 231         if (readMetadataRequest.isBrowsingRequest()) {
 232             remoteRepositories.addAll(repositoryRegistry.getRemoteRepositories());
 233         }
 234
 235         ModelBuildingRequest req =
 236                 new DefaultModelBuildingRequest().setProcessPlugins(false).setPomFile(file.getFilePath().toFile()).setTwoPhaseBuilding(
 237                         false).setValidationLevel(ModelBuildingRequest.VALIDATION_LEVEL_MINIMAL);
 238
 239         //MRM-1607. olamy this will resolve jdk profiles on the current running archiva jvm
 240         req.setSystemProperties(System.getProperties());
 241
 242         // MRM-1411
 243         req.setModelResolver(
 244                 new RepositoryModelResolver(managedRepository, pathTranslator, wagonFactory, remoteRepositories,
 245                         networkProxies, managedRepository, mavenSystemManager, metadataReader));
 246
 247         Model model;
 248         try {
 249             model = builder.build(req).getEffectiveModel();
 250         } catch (ModelBuildingException e) {
 251             String msg = "The artifact's POM file '" + file + "' was invalid: " + e.getMessage();
 252
 253             List<ModelProblem> modelProblems = e.getProblems();
 254             for (ModelProblem problem : modelProblems) {
 255                 // MRM-1411, related to MRM-1335
 256                 // this means that the problem was that the parent wasn't resolved!
 257                 // olamy really hackhish but fail with java profile so use error message
 258                 // || ( StringUtils.startsWith( problem.getMessage(), "Failed to determine Java version for profile" ) )
 259                 // but setTwoPhaseBuilding(true) fix that
 260                 if (((problem.getException() instanceof FileNotFoundException
 261                         || problem.getException() instanceof NoSuchFileException
 262                 ) && e.getModelId() != null &&
 263                         !e.getModelId().equals(problem.getModelId()))) {
 264                     log.warn("The artifact's parent POM file '{}' cannot be resolved. "
 265                             + "Using defaults for project version metadata..", file);
 266
 267                     ProjectVersionMetadata metadata = new ProjectVersionMetadata();
 268                     metadata.setId(readMetadataRequest.getProjectVersion());
 269
 270                     MavenProjectFacet facet = new MavenProjectFacet();
 271                     facet.setGroupId(readMetadataRequest.getNamespace());
 272                     facet.setArtifactId(readMetadataRequest.getProjectId());
 273                     facet.setPackaging("jar");
 274                     metadata.addFacet(facet);
 275
 276                     String errMsg =
 277                             "Error in resolving artifact's parent POM file. " + (problem.getException() == null
 278                                     ? problem.getMessage()
 279                                     : problem.getException().getMessage());
 280                     RepositoryProblemFacet repoProblemFacet = new RepositoryProblemFacet();
 281                     repoProblemFacet.setRepositoryId(readMetadataRequest.getRepositoryId());
 282                     repoProblemFacet.setId(readMetadataRequest.getRepositoryId());
 283                     repoProblemFacet.setMessage(errMsg);
 284                     repoProblemFacet.setProblem(errMsg);
 285                     repoProblemFacet.setProject(readMetadataRequest.getProjectId());
 286                     repoProblemFacet.setVersion(readMetadataRequest.getProjectVersion());
 287                     repoProblemFacet.setNamespace(readMetadataRequest.getNamespace());
 288
 289                     metadata.addFacet(repoProblemFacet);
 290
 291                     return metadata;
 292                 }
 293             }
 294
 295             throw new RepositoryStorageMetadataInvalidException("invalid-pom", msg, e);
 296         }
 297
 298         // Check if the POM is in the correct location
 299         boolean correctGroupId = readMetadataRequest.getNamespace().equals(model.getGroupId());
 300         boolean correctArtifactId = readMetadataRequest.getProjectId().equals(model.getArtifactId());
 301         boolean correctVersion = readMetadataRequest.getProjectVersion().equals(model.getVersion());
 302         if (!correctGroupId || !correctArtifactId || !correctVersion) {
 303             StringBuilder message = new StringBuilder("Incorrect POM coordinates in '" + file + "':");
 304             if (!correctGroupId) {
 305                 message.append("\nIncorrect group ID: ").append(model.getGroupId());
 306             }
 307             if (!correctArtifactId) {
 308                 message.append("\nIncorrect artifact ID: ").append(model.getArtifactId());
 309             }
 310             if (!correctVersion) {
 311                 message.append("\nIncorrect version: ").append(model.getVersion());
 312             }
 313
 314             throw new RepositoryStorageMetadataInvalidException("mislocated-pom", message.toString());
 315         }
 316
 317         ProjectVersionMetadata metadata = new ProjectVersionMetadata();
 318         metadata.setCiManagement(convertCiManagement(model.getCiManagement()));
 319         metadata.setDescription(model.getDescription());
 320         metadata.setId(readMetadataRequest.getProjectVersion());
 321         metadata.setIssueManagement(convertIssueManagement(model.getIssueManagement()));
 322         metadata.setLicenses(convertLicenses(model.getLicenses()));
 323         metadata.setMailingLists(convertMailingLists(model.getMailingLists()));
 324         metadata.setDependencies(convertDependencies(model.getDependencies()));
 325         metadata.setName(model.getName());
 326         metadata.setOrganization(convertOrganization(model.getOrganization()));
 327         metadata.setScm(convertScm(model.getScm()));
 328         metadata.setUrl(model.getUrl());
 329         metadata.setProperties(model.getProperties());
 330
 331         MavenProjectFacet facet = new MavenProjectFacet();
 332         facet.setGroupId(model.getGroupId() != null ? model.getGroupId() : model.getParent().getGroupId());
 333         facet.setArtifactId(model.getArtifactId());
 334         facet.setPackaging(model.getPackaging());
 335         if (model.getParent() != null) {
 336             MavenProjectParent parent = new MavenProjectParent();
 337             parent.setGroupId(model.getParent().getGroupId());
 338             parent.setArtifactId(model.getParent().getArtifactId());
 339             parent.setVersion(model.getParent().getVersion());
 340             facet.setParent(parent);
 341         }
 342         metadata.addFacet(facet);
 343
 344         return metadata;
 345
 346
 347     }
 348
 349     public void setWagonFactory(WagonFactory wagonFactory) {
 350         this.wagonFactory = wagonFactory;
 351     }
 352
 353     private List<org.apache.archiva.metadata.model.Dependency> convertDependencies(List<Dependency> dependencies) {
 354         List<org.apache.archiva.metadata.model.Dependency> l = new ArrayList<>();
 355         for (Dependency dependency : dependencies) {
 356             org.apache.archiva.metadata.model.Dependency newDependency =
 357                     new org.apache.archiva.metadata.model.Dependency();
 358             newDependency.setArtifactId(dependency.getArtifactId());
 359             newDependency.setClassifier(dependency.getClassifier());
 360             newDependency.setNamespace(dependency.getGroupId());
 361             newDependency.setOptional(dependency.isOptional());
 362             newDependency.setScope(dependency.getScope());
 363             newDependency.setSystemPath(dependency.getSystemPath());
 364             newDependency.setType(dependency.getType());
 365             newDependency.setVersion(dependency.getVersion());
 366             l.add(newDependency);
 367         }
 368         return l;
 369     }
 370
 371     private org.apache.archiva.metadata.model.Scm convertScm(Scm scm) {
 372         org.apache.archiva.metadata.model.Scm newScm = null;
 373         if (scm != null) {
 374             newScm = new org.apache.archiva.metadata.model.Scm();
 375             newScm.setConnection(scm.getConnection());
 376             newScm.setDeveloperConnection(scm.getDeveloperConnection());
 377             newScm.setUrl(scm.getUrl());
 378         }
 379         return newScm;
 380     }
 381
 382     private org.apache.archiva.metadata.model.Organization convertOrganization(Organization organization) {
 383         org.apache.archiva.metadata.model.Organization org = null;
 384         if (organization != null) {
 385             org = new org.apache.archiva.metadata.model.Organization();
 386             org.setName(organization.getName());
 387             org.setUrl(organization.getUrl());
 388         }
 389         return org;
 390     }
 391
 392     private List<org.apache.archiva.metadata.model.License> convertLicenses(List<License> licenses) {
 393         List<org.apache.archiva.metadata.model.License> l = new ArrayList<>();
 394         for (License license : licenses) {
 395             org.apache.archiva.metadata.model.License newLicense = new org.apache.archiva.metadata.model.License();
 396             newLicense.setName(license.getName());
 397             newLicense.setUrl(license.getUrl());
 398             l.add(newLicense);
 399         }
 400         return l;
 401     }
 402
 403     private List<org.apache.archiva.metadata.model.MailingList> convertMailingLists(List<MailingList> mailingLists) {
 404         List<org.apache.archiva.metadata.model.MailingList> l = new ArrayList<>();
 405         for (MailingList mailingList : mailingLists) {
 406             org.apache.archiva.metadata.model.MailingList newMailingList =
 407                     new org.apache.archiva.metadata.model.MailingList();
 408             newMailingList.setName(mailingList.getName());
 409             newMailingList.setMainArchiveUrl(mailingList.getArchive());
 410             newMailingList.setPostAddress(mailingList.getPost());
 411             newMailingList.setSubscribeAddress(mailingList.getSubscribe());
 412             newMailingList.setUnsubscribeAddress(mailingList.getUnsubscribe());
 413             newMailingList.setOtherArchives(mailingList.getOtherArchives());
 414             l.add(newMailingList);
 415         }
 416         return l;
 417     }
 418
 419     private org.apache.archiva.metadata.model.IssueManagement convertIssueManagement(IssueManagement issueManagement) {
 420         org.apache.archiva.metadata.model.IssueManagement im = null;
 421         if (issueManagement != null) {
 422             im = new org.apache.archiva.metadata.model.IssueManagement();
 423             im.setSystem(issueManagement.getSystem());
 424             im.setUrl(issueManagement.getUrl());
 425         }
 426         return im;
 427     }
 428
 429     private org.apache.archiva.metadata.model.CiManagement convertCiManagement(CiManagement ciManagement) {
 430         org.apache.archiva.metadata.model.CiManagement ci = null;
 431         if (ciManagement != null) {
 432             ci = new org.apache.archiva.metadata.model.CiManagement();
 433             ci.setSystem(ciManagement.getSystem());
 434             ci.setUrl(ciManagement.getUrl());
 435         }
 436         return ci;
 437     }
 438
 439     @Override
 440     public Collection<String> listRootNamespaces(String repoId, Filter<String> filter)
 441             throws RepositoryStorageRuntimeException {
 442         StorageAsset dir = getRepositoryBasedir(repoId);
 443
 444         return getSortedFiles(dir, filter);
 445     }
 446
 447     private static Collection<String> getSortedFiles(StorageAsset dir, Filter<String> filter) {
 448
 449         final Predicate<StorageAsset> dFilter = new DirectoryFilter(filter);
 450         return dir.list().stream().filter(f -> f.isContainer())
 451                 .filter(dFilter)
 452                 .map(path -> path.getName().toString())
 453                 .sorted().collect(Collectors.toList());
 454
 455     }
 456
 457     private StorageAsset getRepositoryBasedir(String repoId)
 458             throws RepositoryStorageRuntimeException {
 459         ManagedRepository repository = repositoryRegistry.getManagedRepository(repoId);
 460
 461         return repository.getRoot();
 462     }
 463
 464     @Override
 465     public Collection<String> listNamespaces(String repoId, String namespace, Filter<String> filter)
 466             throws RepositoryStorageRuntimeException {
 467         StorageAsset dir = pathTranslator.toFile(getRepositoryBasedir(repoId), namespace);
 468         if (!(dir.exists()) && !dir.isContainer()) {
 469             return Collections.emptyList();
 470         }
 471         // scan all the directories which are potential namespaces. Any directories known to be projects are excluded
 472         Predicate<StorageAsset> dFilter = new DirectoryFilter(filter);
 473         return dir.list().stream().filter(dFilter).filter(path -> !isProject(path, filter)).map(path -> path.getName().toString())
 474                 .sorted().collect(Collectors.toList());
 475     }
 476
 477     @Override
 478     public Collection<String> listProjects(String repoId, String namespace, Filter<String> filter)
 479             throws RepositoryStorageRuntimeException {
 480         StorageAsset dir = pathTranslator.toFile(getRepositoryBasedir(repoId), namespace);
 481         if (!(dir.exists() && dir.isContainer())) {
 482             return Collections.emptyList();
 483         }
 484         // scan all directories in the namespace, and only include those that are known to be projects
 485         final Predicate<StorageAsset> dFilter = new DirectoryFilter(filter);
 486         return dir.list().stream().filter(dFilter).filter(path -> isProject(path, filter)).map(path -> path.getName().toString())
 487                 .sorted().collect(Collectors.toList());
 488
 489     }
 490
 491     @Override
 492     public Collection<String> listProjectVersions(String repoId, String namespace, String projectId,
 493                                                   Filter<String> filter)
 494             throws RepositoryStorageRuntimeException {
 495         StorageAsset dir = pathTranslator.toFile(getRepositoryBasedir(repoId), namespace, projectId);
 496         if (!(dir.exists() && dir.isContainer())) {
 497             return Collections.emptyList();
 498         }
 499
 500         // all directories in a project directory can be considered a version
 501         return getSortedFiles(dir, filter);
 502     }
 503
 504     @Override
 505     public Collection<ArtifactMetadata> readArtifactsMetadata(ReadMetadataRequest readMetadataRequest)
 506             throws RepositoryStorageRuntimeException {
 507         StorageAsset dir = pathTranslator.toFile(getRepositoryBasedir(readMetadataRequest.getRepositoryId()),
 508                 readMetadataRequest.getNamespace(), readMetadataRequest.getProjectId(),
 509                 readMetadataRequest.getProjectVersion());
 510         if (!(dir.exists() && dir.isContainer())) {
 511             return Collections.emptyList();
 512         }
 513
 514         // all files that are not metadata and not a checksum / signature are considered artifacts
 515         final Predicate<StorageAsset> dFilter = new ArtifactDirectoryFilter(readMetadataRequest.getFilter());
 516         // Returns a map TRUE -> (success values), FALSE -> (Exceptions)
 517         Map<Boolean, List<Try<ArtifactMetadata>>> result = dir.list().stream().filter(dFilter).map(path -> {
 518                     try {
 519                         return Try.success(getArtifactFromFile(readMetadataRequest.getRepositoryId(), readMetadataRequest.getNamespace(),
 520                                 readMetadataRequest.getProjectId(), readMetadataRequest.getProjectVersion(),
 521                                 path));
 522                     } catch (Exception e) {
 523                         log.debug("Could not create metadata for {}:  {}", path, e.getMessage(), e);
 524                         return Try.<ArtifactMetadata>failure(e);
 525                     }
 526                 }
 527         ).collect(Collectors.groupingBy(Try::isSuccess));
 528         if (result.containsKey(Boolean.FALSE) && result.get(Boolean.FALSE).size() > 0 && (!result.containsKey(Boolean.TRUE) || result.get(Boolean.TRUE).size() == 0)) {
 529             log.error("Could not get artifact metadata. Directory: {}. Number of errors {}.", dir, result.get(Boolean.FALSE).size());
 530             Try<ArtifactMetadata> failure = result.get(Boolean.FALSE).get(0);
 531             log.error("Sample exception {}", failure.getError().getMessage(), failure.getError());
 532             throw new RepositoryStorageRuntimeException(readMetadataRequest.getRepositoryId(), "Could not retrieve metadata of the files");
 533         } else {
 534             if (!result.containsKey(Boolean.TRUE) || result.get(Boolean.TRUE) == null) {
 535                 return Collections.emptyList();
 536             }
 537             return result.get(Boolean.TRUE).stream().map(tr -> tr.get()).collect(Collectors.toList());
 538         }
 539
 540     }
 541
 542     @Override
 543     public ArtifactMetadata readArtifactMetadataFromPath(String repoId, String path)
 544             throws RepositoryStorageRuntimeException {
 545         ArtifactMetadata metadata = pathTranslator.getArtifactForPath(repoId, path);
 546
 547         try {
 548             populateArtifactMetadataFromFile(metadata, getRepositoryBasedir(repoId).resolve(path));
 549         } catch (IOException e) {
 550             throw new RepositoryStorageRuntimeException(repoId, "Error during metadata retrieval of " + path + " :" + e.getMessage(), e);
 551         }
 552
 553         return metadata;
 554     }
 555
 556     private ArtifactMetadata getArtifactFromFile(String repoId, String namespace, String projectId,
 557                                                  String projectVersion, StorageAsset file) throws IOException {
 558         ArtifactMetadata metadata =
 559                 pathTranslator.getArtifactFromId(repoId, namespace, projectId, projectVersion, file.getName());
 560
 561         populateArtifactMetadataFromFile(metadata, file);
 562
 563         return metadata;
 564     }
 565
 566     @Override
 567     public void applyServerSideRelocation(ManagedRepository managedRepository, ArtifactReference artifact)
 568             throws ProxyDownloadException {
 569         if ("pom".equals(artifact.getType())) {
 570             return;
 571         }
 572
 573         // Build the artifact POM reference
 574         ArtifactReference pomReference = new ArtifactReference();
 575         pomReference.setGroupId(artifact.getGroupId());
 576         pomReference.setArtifactId(artifact.getArtifactId());
 577         pomReference.setVersion(artifact.getVersion());
 578         pomReference.setProjectVersion( artifact.getProjectVersion() );
 579         pomReference.setType("pom");
 580         BaseRepositoryContentLayout layout;
 581         try
 582         {
 583             layout = managedRepository.getContent( ).getLayout( BaseRepositoryContentLayout.class );
 584         }
 585         catch ( LayoutException e )
 586         {
 587             throw new ProxyDownloadException( "Could not set layout " + e.getMessage( ), new HashMap<>(  ) );
 588         }
 589
 590         RepositoryType repositoryType = managedRepository.getType();
 591         if (!proxyRegistry.hasHandler(repositoryType)) {
 592             throw new ProxyDownloadException("No proxy handler found for repository type " + repositoryType, new HashMap<>());
 593         }
 594
 595         ItemSelector selector = ArchivaItemSelector.builder( )
 596             .withNamespace( artifact.getGroupId( ) )
 597             .withProjectId( artifact.getArtifactId( ) )
 598             .withArtifactId( artifact.getArtifactId( ) )
 599             .withVersion( artifact.getVersion( ) )
 600             .withArtifactVersion( artifact.getVersion( ) )
 601             .withType( "pom" ).build( );
 602
 603         Artifact pom = layout.getArtifact( selector );
 604
 605         RepositoryProxyHandler proxyHandler = proxyRegistry.getHandler(repositoryType).get(0);
 606
 607         // Get the artifact POM from proxied repositories if needed
 608         proxyHandler.fetchFromProxies(managedRepository, pomReference);
 609
 610         // Open and read the POM from the managed repo
 611
 612         if (!pom.exists()) {
 613             return;
 614         }
 615
 616         try {
 617             // MavenXpp3Reader leaves the file open, so we need to close it ourselves.
 618
 619             Model model;
 620             try (Reader reader = Channels.newReader(pom.getAsset().getReadChannel(), Charset.defaultCharset().name())) {
 621                 model = MAVEN_XPP_3_READER.read(reader);
 622             }
 623
 624             DistributionManagement dist = model.getDistributionManagement();
 625             if (dist != null) {
 626                 Relocation relocation = dist.getRelocation();
 627                 if (relocation != null) {
 628                     // artifact is relocated : update the repositoryPath
 629                     if (relocation.getGroupId() != null) {
 630                         artifact.setGroupId(relocation.getGroupId());
 631                     }
 632                     if (relocation.getArtifactId() != null) {
 633                         artifact.setArtifactId(relocation.getArtifactId());
 634                     }
 635                     if (relocation.getVersion() != null) {
 636                         artifact.setVersion(relocation.getVersion());
 637                     }
 638                 }
 639             }
 640         } catch (IOException e) {
 641             // Unable to read POM : ignore.
 642         } catch (XmlPullParserException e) {
 643             // Invalid POM : ignore
 644         }
 645     }
 646
 647     @Override
 648     public ItemSelector applyServerSideRelocation(ManagedRepository managedRepository, ItemSelector artifactSelector)
 649         throws ProxyDownloadException {
 650         if ("pom".equals(artifactSelector.getType())) {
 651             return artifactSelector;
 652         }
 653
 654         // Build the artifact POM reference
 655         BaseRepositoryContentLayout layout;
 656         try
 657         {
 658             layout = managedRepository.getContent( ).getLayout( BaseRepositoryContentLayout.class );
 659         }
 660         catch ( LayoutException e )
 661         {
 662             throw new ProxyDownloadException( "Could not set layout " + e.getMessage( ), new HashMap<>(  ) );
 663         }
 664
 665         RepositoryType repositoryType = managedRepository.getType();
 666         if (!proxyRegistry.hasHandler(repositoryType)) {
 667             throw new ProxyDownloadException("No proxy handler found for repository type " + repositoryType, new HashMap<>());
 668         }
 669
 670
 671
 672         ItemSelector selector = ArchivaItemSelector.builder( )
 673             .withNamespace( artifactSelector.getNamespace( ) )
 674             .withProjectId( artifactSelector.getArtifactId( ) )
 675             .withArtifactId( artifactSelector.getArtifactId( ) )
 676             .withVersion( artifactSelector.getVersion( ) )
 677             .withArtifactVersion( artifactSelector.getVersion( ) )
 678             .withType( "pom" ).build( );
 679
 680         Artifact pom = layout.getArtifact( selector );
 681
 682         RepositoryProxyHandler proxyHandler = proxyRegistry.getHandler(repositoryType).get(0);
 683
 684         // Get the artifact POM from proxied repositories if needed
 685         proxyHandler.fetchFromProxies(managedRepository, pom);
 686
 687         // Open and read the POM from the managed repo
 688
 689         if (!pom.exists()) {
 690             return artifactSelector;
 691         }
 692
 693         try {
 694             // MavenXpp3Reader leaves the file open, so we need to close it ourselves.
 695
 696             Model model;
 697             try (Reader reader = Channels.newReader(pom.getAsset().getReadChannel(), Charset.defaultCharset().name())) {
 698                 model = MAVEN_XPP_3_READER.read(reader);
 699             }
 700
 701             DistributionManagement dist = model.getDistributionManagement();
 702             if (dist != null) {
 703                 Relocation relocation = dist.getRelocation();
 704                 if (relocation != null) {
 705                     ArchivaItemSelector.Builder relocatedBuilder = ArchivaItemSelector.builder( );
 706                     // artifact is relocated : update the repositoryPath
 707                     if (relocation.getGroupId() != null) {
 708                         relocatedBuilder.withNamespace( relocation.getGroupId( ) );
 709                     } else {
 710                         relocatedBuilder.withNamespace( artifactSelector.getNamespace( ) );
 711                     }
 712                     if (relocation.getArtifactId() != null) {
 713                         relocatedBuilder.withArtifactId( relocation.getArtifactId( ) );
 714                     } else {
 715                         relocatedBuilder.withArtifactId( artifactSelector.getArtifactId( ) );
 716                     }
 717                     if (relocation.getVersion() != null)
 718                     {
 719                         relocatedBuilder.withVersion( relocation.getVersion( ) );
 720                     } else {
 721                         relocatedBuilder.withVersion( artifactSelector.getVersion( ) );
 722                     }
 723                     return relocatedBuilder.withArtifactVersion( artifactSelector.getArtifactVersion( ) )
 724                         .withClassifier( artifactSelector.getClassifier( ) )
 725                         .withType( artifactSelector.getType( ) )
 726                         .withProjectId( artifactSelector.getProjectId( ) )
 727                         .withExtension( artifactSelector.getExtension( ) )
 728                         .build( );
 729                 }
 730             }
 731         } catch (IOException e) {
 732             // Unable to read POM : ignore.
 733         } catch (XmlPullParserException e) {
 734             // Invalid POM : ignore
 735         }
 736         return artifactSelector;
 737     }
 738
 739
 740     @Override
 741     public String getFilePath(String requestPath, org.apache.archiva.repository.ManagedRepository managedRepository) {
 742         // managedRepository can be null
 743         // extract artifact reference from url
 744         // groupId:artifactId:version:packaging:classifier
 745         //org/apache/archiva/archiva-checksum/1.4-M4-SNAPSHOT/archiva-checksum-1.4-M4-SNAPSHOT.jar
 746         String logicalResource = null;
 747         String requestPathInfo = StringUtils.defaultString(requestPath);
 748
 749         //remove prefix ie /repository/blah becomes /blah
 750         requestPathInfo = removePrefix(requestPathInfo);
 751
 752         // Remove prefixing slash as the repository id doesn't contain it;
 753         if (requestPathInfo.startsWith("/")) {
 754             requestPathInfo = requestPathInfo.substring(1);
 755         }
 756
 757         int slash = requestPathInfo.indexOf('/');
 758         if (slash > 0) {
 759             logicalResource = requestPathInfo.substring(slash);
 760
 761             if (logicalResource.endsWith("/..")) {
 762                 logicalResource += "/";
 763             }
 764
 765             if (logicalResource != null && logicalResource.startsWith("//")) {
 766                 logicalResource = logicalResource.substring(1);
 767             }
 768
 769             if (logicalResource == null) {
 770                 logicalResource = "/";
 771             }
 772         } else {
 773             logicalResource = "/";
 774         }
 775         return logicalResource;
 776
 777     }
 778
 779     @Override
 780     public String getFilePathWithVersion(final String requestPath, ManagedRepositoryContent managedRepositoryContent)
 781             throws RelocationException
 782     {
 783
 784         if (StringUtils.endsWith(requestPath, METADATA_FILENAME)) {
 785             return getFilePath(requestPath, managedRepositoryContent.getRepository());
 786         }
 787
 788         String filePath = getFilePath(requestPath, managedRepositoryContent.getRepository());
 789
 790         ArtifactReference artifactReference = null;
 791         try {
 792             artifactReference = pathParser.toArtifactReference(filePath);
 793         } catch (LayoutException e) {
 794             return filePath;
 795         }
 796
 797         if (StringUtils.endsWith(artifactReference.getVersion(), VersionUtil.SNAPSHOT)) {
 798             // read maven metadata to get last timestamp
 799             StorageAsset metadataDir = managedRepositoryContent.getRepository().getAsset(filePath).getParent();
 800             if (!metadataDir.exists()) {
 801                 return filePath;
 802             }
 803             StorageAsset metadataFile = metadataDir.resolve(METADATA_FILENAME);
 804             if (!metadataFile.exists()) {
 805                 return filePath;
 806             }
 807             ArchivaRepositoryMetadata archivaRepositoryMetadata = null;
 808             try
 809             {
 810                 archivaRepositoryMetadata = metadataReader.read(metadataFile);
 811             }
 812             catch ( RepositoryMetadataException e )
 813             {
 814                 log.error( "Could not read metadata {}", e.getMessage( ), e );
 815                 return filePath;
 816             }
 817             int buildNumber = archivaRepositoryMetadata.getSnapshotVersion().getBuildNumber();
 818             String timestamp = archivaRepositoryMetadata.getSnapshotVersion().getTimestamp();
 819
 820             // MRM-1846
 821             if (buildNumber < 1 && timestamp == null) {
 822                 return filePath;
 823             }
 824
 825             // org/apache/archiva/archiva-checksum/1.4-M4-SNAPSHOT/archiva-checksum-1.4-M4-SNAPSHOT.jar
 826             // ->  archiva-checksum-1.4-M4-20130425.081822-1.jar
 827
 828             filePath = StringUtils.replace(filePath, //
 829                     artifactReference.getArtifactId() //
 830                             + "-" + artifactReference.getVersion(), //
 831                     artifactReference.getArtifactId() //
 832                             + "-" + StringUtils.remove(artifactReference.getVersion(),
 833                             "-" + VersionUtil.SNAPSHOT) //
 834                             + "-" + timestamp //
 835                             + "-" + buildNumber);
 836
 837             throw new RelocationException("/repository/" + managedRepositoryContent.getRepository().getId() +
 838                     (StringUtils.startsWith(filePath, "/") ? "" : "/") + filePath,
 839                     RelocationException.RelocationType.TEMPORARY);
 840
 841         }
 842
 843         return filePath;
 844     }
 845
 846     //-----------------------------
 847     // internal
 848     //-----------------------------
 849
 850     /**
 851      * FIXME remove
 852      *
 853      * @param href
 854      * @return
 855      */
 856     private static String removePrefix(final String href) {
 857         String[] parts = StringUtils.split(href, '/');
 858         parts = (String[]) ArrayUtils.subarray(parts, 1, parts.length);
 859         if (parts == null || parts.length == 0) {
 860             return "/";
 861         }
 862
 863         String joinedString = StringUtils.join(parts, '/');
 864         if (href.endsWith("/")) {
 865             joinedString = joinedString + "/";
 866         }
 867
 868         return joinedString;
 869     }
 870
 871     private static void populateArtifactMetadataFromFile(ArtifactMetadata metadata, StorageAsset file) throws IOException {
 872         metadata.setWhenGathered(ZonedDateTime.now(ZoneId.of("GMT")));
 873         metadata.setFileLastModified(file.getModificationTime().toEpochMilli());
 874         ChecksummedFile checksummedFile = new ChecksummedFile(file.getFilePath());
 875         try {
 876             metadata.setMd5(checksummedFile.calculateChecksum(ChecksumAlgorithm.MD5));
 877         } catch (IOException e) {
 878             log.error("Unable to checksum file {}: {},MD5", file, e.getMessage());
 879         }
 880         try {
 881             metadata.setSha1(checksummedFile.calculateChecksum(ChecksumAlgorithm.SHA1));
 882         } catch (IOException e) {
 883             log.error("Unable to checksum file {}: {},SHA1", file, e.getMessage());
 884         }
 885         metadata.setSize(file.getSize());
 886     }
 887
 888     private boolean isProject(StorageAsset dir, Filter<String> filter) {
 889         // scan directories for a valid project version subdirectory, meaning this must be a project directory
 890         final Predicate<StorageAsset> dFilter = new DirectoryFilter(filter);
 891         boolean projFound = dir.list().stream().filter(dFilter)
 892                 .anyMatch(path -> isProjectVersion(path));
 893         if (projFound) {
 894             return true;
 895         }
 896
 897         // if a metadata file is present, check if this is the "artifactId" directory, marking it as a project
 898         ArchivaRepositoryMetadata metadata = readMetadata(dir);
 899         if (metadata != null && dir.getName().toString().equals(metadata.getArtifactId())) {
 900             return true;
 901         }
 902
 903         return false;
 904     }
 905
 906     private boolean isProjectVersion(StorageAsset dir) {
 907         final String artifactId = dir.getParent().getName();
 908         final String projectVersion = dir.getName();
 909
 910         // check if there is a POM artifact file to ensure it is a version directory
 911
 912         Predicate<StorageAsset> filter;
 913         if (VersionUtil.isSnapshot(projectVersion)) {
 914             filter = new PomFilenameFilter(artifactId, projectVersion);
 915         } else {
 916             final String pomFile = artifactId + "-" + projectVersion + ".pom";
 917             filter = new PomFileFilter(pomFile);
 918         }
 919         if (dir.list().stream().filter(f -> !f.isContainer()).anyMatch(filter)) {
 920             return true;
 921         }
 922         // if a metadata file is present, check if this is the "version" directory, marking it as a project version
 923         ArchivaRepositoryMetadata metadata = readMetadata(dir);
 924         if (metadata != null && projectVersion.equals(metadata.getVersion())) {
 925             return true;
 926         }
 927
 928         return false;
 929     }
 930
 931     private ArchivaRepositoryMetadata readMetadata(StorageAsset directory) {
 932         ArchivaRepositoryMetadata metadata = null;
 933         StorageAsset metadataFile = directory.resolve(METADATA_FILENAME);
 934         if (metadataFile.exists()) {
 935             try {
 936                 metadata = metadataReader.read(metadataFile);
 937             } catch ( RepositoryMetadataException e )
 938             {
 939                 // Ignore missing or invalid metadata
 940             }
 941         }
 942         return metadata;
 943     }
 944
 945     private static class DirectoryFilter
 946             implements Predicate<StorageAsset> {
 947         private final Filter<String> filter;
 948
 949         public DirectoryFilter(Filter<String> filter) {
 950             this.filter = filter;
 951         }
 952
 953         @Override
 954         public boolean test(StorageAsset dir) {
 955             final String name = dir.getName();
 956             if (!filter.accept(name)) {
 957                 return false;
 958             } else if (name.startsWith(".")) {
 959                 return false;
 960             } else if (!dir.isContainer()) {
 961                 return false;
 962             }
 963             return true;
 964         }
 965     }
 966
 967     private static class ArtifactDirectoryFilter
 968             implements Predicate<StorageAsset> {
 969         private final Filter<String> filter;
 970
 971         private ArtifactDirectoryFilter(Filter<String> filter) {
 972             this.filter = filter;
 973         }
 974
 975         @Override
 976         public boolean test(StorageAsset file) {
 977             final Set<String> checksumExts = ChecksumAlgorithm.getAllExtensions();
 978             final String path = file.getPath();
 979             final String name = file.getName();
 980             final String extension = StringUtils.substringAfterLast(name, ".").toLowerCase();
 981             // TODO compare to logic in maven-repository-layer
 982             if (file.isContainer()) {
 983                 return false;
 984             } else if (!filter.accept(name)) {
 985                 return false;
 986             } else if (name.startsWith(".") || path.contains("/.") ) {
 987                 return false;
 988             } else if (checksumExts.contains(extension)) {
 989                 return false;
 990             } else if (Arrays.binarySearch(IGNORED_FILES, name) >= 0) {
 991                 return false;
 992             }
 993             // some files from remote repositories can have name like maven-metadata-archiva-vm-all-public.xml
 994             else if (StringUtils.startsWith(name, METADATA_FILENAME_START) && StringUtils.endsWith(name, ".xml")) {
 995                 return false;
 996             }
 997
 998             return true;
 999
1000         }
1001     }
1002
1003
1004     private static final class PomFilenameFilter
1005             implements Predicate<StorageAsset> {
1006
1007         private final String artifactId, projectVersion;
1008
1009         private PomFilenameFilter(String artifactId, String projectVersion) {
1010             this.artifactId = artifactId;
1011             this.projectVersion = projectVersion;
1012         }
1013
1014         @Override
1015         public boolean test(StorageAsset dir) {
1016             final String name = dir.getName();
1017             if (name.startsWith(artifactId + "-") && name.endsWith(".pom")) {
1018                 String v = name.substring(artifactId.length() + 1, name.length() - 4);
1019                 v = VersionUtil.getBaseVersion(v);
1020                 if (v.equals(projectVersion)) {
1021                     return true;
1022                 }
1023             }
1024             return false;
1025         }
1026
1027     }
1028
1029     private static class PomFileFilter
1030             implements Predicate<StorageAsset> {
1031         private final String pomFile;
1032
1033         private PomFileFilter(String pomFile) {
1034             this.pomFile = pomFile;
1035         }
1036
1037         @Override
1038         public boolean test(StorageAsset dir) {
1039             return pomFile.equals(dir.getName());
1040         }
1041     }
1042
1043
1044     public PathParser getPathParser() {
1045         return pathParser;
1046     }
1047
1048     public void setPathParser(PathParser pathParser) {
1049         this.pathParser = pathParser;
1050     }
1051 }