source.dussan.org Git - archiva.git/blob

   1 package org.apache.archiva.metadata.repository.storage.maven2;
   2
   3 /*
   4  * Licensed to the Apache Software Foundation (ASF) under one
   5  * or more contributor license agreements.  See the NOTICE file
   6  * distributed with this work for additional information
   7  * regarding copyright ownership.  The ASF licenses this file
   8  * to you under the Apache License, Version 2.0 (the
   9  * "License"); you may not use this file except in compliance
  10  * with the License.  You may obtain a copy of the License at
  11  *
  12  *   http://www.apache.org/licenses/LICENSE-2.0
  13  *
  14  * Unless required by applicable law or agreed to in writing,
  15  * software distributed under the License is distributed on an
  16  * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
  17  * KIND, either express or implied.  See the License for the
  18  * specific language governing permissions and limitations
  19  * under the License.
  20  */
  21
  22 import org.apache.archiva.checksum.ChecksumAlgorithm;
  23 import org.apache.archiva.checksum.ChecksummedFile;
  24 import org.apache.archiva.common.Try;
  25 import org.apache.archiva.common.utils.VersionUtil;
  26 import org.apache.archiva.maven2.metadata.MavenMetadataReader;
  27 import org.apache.archiva.metadata.model.ArtifactMetadata;
  28 import org.apache.archiva.metadata.model.ProjectMetadata;
  29 import org.apache.archiva.metadata.model.ProjectVersionMetadata;
  30 import org.apache.archiva.metadata.model.facets.RepositoryProblemFacet;
  31 import org.apache.archiva.metadata.repository.filter.Filter;
  32 import org.apache.archiva.metadata.repository.storage.ReadMetadataRequest;
  33 import org.apache.archiva.metadata.repository.storage.RelocationException;
  34 import org.apache.archiva.metadata.repository.storage.RepositoryPathTranslator;
  35 import org.apache.archiva.metadata.repository.storage.RepositoryStorage;
  36 import org.apache.archiva.metadata.repository.storage.RepositoryStorageMetadataInvalidException;
  37 import org.apache.archiva.metadata.repository.storage.RepositoryStorageMetadataNotFoundException;
  38 import org.apache.archiva.metadata.repository.storage.RepositoryStorageRuntimeException;
  39 import org.apache.archiva.model.ArchivaRepositoryMetadata;
  40 import org.apache.archiva.model.ArtifactReference;
  41 import org.apache.archiva.model.SnapshotVersion;
  42 import org.apache.archiva.policies.ProxyDownloadException;
  43 import org.apache.archiva.proxy.ProxyRegistry;
  44 import org.apache.archiva.proxy.maven.WagonFactory;
  45 import org.apache.archiva.proxy.model.NetworkProxy;
  46 import org.apache.archiva.proxy.model.ProxyConnector;
  47 import org.apache.archiva.proxy.model.RepositoryProxyHandler;
  48 import org.apache.archiva.repository.*;
  49 import org.apache.archiva.repository.content.PathParser;
  50 import org.apache.archiva.repository.maven2.MavenSystemManager;
  51 import org.apache.archiva.xml.XMLException;
  52 import org.apache.commons.lang.ArrayUtils;
  53 import org.apache.commons.lang.StringUtils;
  54 import org.apache.maven.model.CiManagement;
  55 import org.apache.maven.model.Dependency;
  56 import org.apache.maven.model.DistributionManagement;
  57 import org.apache.maven.model.IssueManagement;
  58 import org.apache.maven.model.License;
  59 import org.apache.maven.model.MailingList;
  60 import org.apache.maven.model.Model;
  61 import org.apache.maven.model.Organization;
  62 import org.apache.maven.model.Relocation;
  63 import org.apache.maven.model.Scm;
  64 import org.apache.maven.model.building.DefaultModelBuilderFactory;
  65 import org.apache.maven.model.building.DefaultModelBuildingRequest;
  66 import org.apache.maven.model.building.ModelBuilder;
  67 import org.apache.maven.model.building.ModelBuildingException;
  68 import org.apache.maven.model.building.ModelBuildingRequest;
  69 import org.apache.maven.model.building.ModelProblem;
  70 import org.apache.maven.model.io.xpp3.MavenXpp3Reader;
  71 import org.codehaus.plexus.util.xml.pull.XmlPullParserException;
  72 import org.slf4j.Logger;
  73 import org.slf4j.LoggerFactory;
  74 import org.springframework.context.ApplicationContext;
  75 import org.springframework.stereotype.Service;
  76
  77 import javax.annotation.PostConstruct;
  78 import javax.inject.Inject;
  79 import javax.inject.Named;
  80 import java.io.FileNotFoundException;
  81 import java.io.IOException;
  82 import java.io.Reader;
  83 import java.nio.charset.Charset;
  84 import java.nio.file.Files;
  85 import java.nio.file.NoSuchFileException;
  86 import java.nio.file.Path;
  87 import java.nio.file.Paths;
  88 import java.util.ArrayList;
  89 import java.util.Arrays;
  90 import java.util.Collection;
  91 import java.util.Collections;
  92 import java.util.Date;
  93 import java.util.HashMap;
  94 import java.util.List;
  95 import java.util.Map;
  96 import java.util.function.Predicate;
  97 import java.util.stream.Collectors;
  98 import java.util.stream.Stream;
  99
 100 // import java.io.FileNotFoundException;
 101
 102 /**
 103  * <p>
 104  * Maven 2 repository format storage implementation. This class currently takes parameters to indicate the repository to
 105  * deal with rather than being instantiated per-repository.
 106  * FIXME: instantiate one per repository and allocate permanently from a factory (which can be obtained within the session).
 107  * </p>
 108  * <p>
 109  * The session is passed in as an argument to obtain any necessary resources, rather than the class being instantiated
 110  * within the session in the context of a single managed repository's resolution needs.
 111  * </p>
 112  */
 113 @Service("repositoryStorage#maven2")
 114 public class Maven2RepositoryStorage
 115         implements RepositoryStorage {
 116
 117     private static final Logger LOGGER = LoggerFactory.getLogger(Maven2RepositoryStorage.class);
 118
 119     private ModelBuilder builder;
 120
 121     @Inject
 122     RepositoryRegistry repositoryRegistry;
 123
 124     @Inject
 125     @Named("repositoryPathTranslator#maven2")
 126     private RepositoryPathTranslator pathTranslator;
 127
 128     @Inject
 129     private WagonFactory wagonFactory;
 130
 131     @Inject
 132     private ApplicationContext applicationContext;
 133
 134     @Inject
 135     @Named("pathParser#default")
 136     private PathParser pathParser;
 137
 138     @Inject
 139     private ProxyRegistry proxyRegistry;
 140
 141     @Inject
 142     private MavenSystemManager mavenSystemManager;
 143
 144     private static final String METADATA_FILENAME_START = "maven-metadata";
 145
 146     private static final String METADATA_FILENAME = METADATA_FILENAME_START + ".xml";
 147
 148     // This array must be lexically sorted
 149     private static final String[] IGNORED_FILES = {METADATA_FILENAME, "resolver-status.properties"};
 150
 151     private static final MavenXpp3Reader MAVEN_XPP_3_READER = new MavenXpp3Reader();
 152
 153
 154     @PostConstruct
 155     public void initialize() {
 156         builder = new DefaultModelBuilderFactory().newInstance();
 157
 158     }
 159
 160     @Override
 161     public ProjectMetadata readProjectMetadata(String repoId, String namespace, String projectId) {
 162         // TODO: could natively implement the "shared model" concept from the browse action to avoid needing it there?
 163         return null;
 164     }
 165
 166     @Override
 167     public ProjectVersionMetadata readProjectVersionMetadata(ReadMetadataRequest readMetadataRequest)
 168             throws RepositoryStorageMetadataNotFoundException, RepositoryStorageMetadataInvalidException,
 169             RepositoryStorageRuntimeException {
 170
 171         ManagedRepository managedRepository = repositoryRegistry.getManagedRepository(readMetadataRequest.getRepositoryId());
 172         boolean isReleases = managedRepository.getActiveReleaseSchemes().contains(ReleaseScheme.RELEASE);
 173         boolean isSnapshots = managedRepository.getActiveReleaseSchemes().contains(ReleaseScheme.SNAPSHOT);
 174         String artifactVersion = readMetadataRequest.getProjectVersion();
 175         // olamy: in case of browsing via the ui we can mix repos (parent of a SNAPSHOT can come from release repo)
 176         if (!readMetadataRequest.isBrowsingRequest()) {
 177             if (VersionUtil.isSnapshot(artifactVersion)) {
 178                 // skygo trying to improve speed by honoring managed configuration MRM-1658
 179                 if (isReleases && !isSnapshots) {
 180                     throw new RepositoryStorageRuntimeException("lookforsnaponreleaseonly",
 181                             "managed repo is configured for release only");
 182                 }
 183             } else {
 184                 if (!isReleases && isSnapshots) {
 185                     throw new RepositoryStorageRuntimeException("lookforsreleaseonsneponly",
 186                             "managed repo is configured for snapshot only");
 187                 }
 188             }
 189         }
 190         Path basedir = Paths.get(managedRepository.getLocation());
 191         if (VersionUtil.isSnapshot(artifactVersion)) {
 192             Path metadataFile = pathTranslator.toFile(basedir, readMetadataRequest.getNamespace(),
 193                     readMetadataRequest.getProjectId(), artifactVersion,
 194                     METADATA_FILENAME);
 195             try {
 196                 ArchivaRepositoryMetadata metadata = MavenMetadataReader.read(metadataFile);
 197
 198                 // re-adjust to timestamp if present, otherwise retain the original -SNAPSHOT filename
 199                 SnapshotVersion snapshotVersion = metadata.getSnapshotVersion();
 200                 if (snapshotVersion != null) {
 201                     artifactVersion =
 202                             artifactVersion.substring(0, artifactVersion.length() - 8); // remove SNAPSHOT from end
 203                     artifactVersion =
 204                             artifactVersion + snapshotVersion.getTimestamp() + "-" + snapshotVersion.getBuildNumber();
 205                 }
 206             } catch (XMLException e) {
 207                 // unable to parse metadata - LOGGER it, and continue with the version as the original SNAPSHOT version
 208                 LOGGER.warn("Invalid metadata: {} - {}", metadataFile, e.getMessage());
 209             }
 210         }
 211
 212         // TODO: won't work well with some other layouts, might need to convert artifact parts to ID by path translator
 213         String id = readMetadataRequest.getProjectId() + "-" + artifactVersion + ".pom";
 214         Path file =
 215                 pathTranslator.toFile(basedir, readMetadataRequest.getNamespace(), readMetadataRequest.getProjectId(),
 216                         readMetadataRequest.getProjectVersion(), id);
 217
 218         if (!Files.exists(file)) {
 219             // metadata could not be resolved
 220             throw new RepositoryStorageMetadataNotFoundException(
 221                     "The artifact's POM file '" + file.toAbsolutePath() + "' was missing");
 222         }
 223
 224         // TODO: this is a workaround until we can properly resolve using proxies as well - this doesn't cache
 225         //       anything locally!
 226         List<RemoteRepository> remoteRepositories = new ArrayList<>();
 227         Map<String, NetworkProxy> networkProxies = new HashMap<>();
 228
 229         Map<String, List<ProxyConnector>> proxyConnectorsMap = proxyRegistry.getProxyConnectorAsMap();
 230         List<ProxyConnector> proxyConnectors = proxyConnectorsMap.get(readMetadataRequest.getRepositoryId());
 231         if (proxyConnectors != null) {
 232             for (ProxyConnector proxyConnector : proxyConnectors) {
 233                 RemoteRepository remoteRepoConfig =
 234                         repositoryRegistry.getRemoteRepository(proxyConnector.getTargetRepository().getId());
 235
 236                 if (remoteRepoConfig != null) {
 237                     remoteRepositories.add(remoteRepoConfig);
 238
 239                     NetworkProxy networkProxyConfig =
 240                             proxyRegistry.getNetworkProxy(proxyConnector.getProxyId());
 241
 242                     if (networkProxyConfig != null) {
 243                         // key/value: remote repo ID/proxy info
 244                         networkProxies.put(proxyConnector.getTargetRepository().getId(), networkProxyConfig);
 245                     }
 246                 }
 247             }
 248         }
 249
 250         // That's a browsing request so we can a mix of SNAPSHOT and release artifacts (especially with snapshots which
 251         // can have released parent pom
 252         if (readMetadataRequest.isBrowsingRequest()) {
 253             remoteRepositories.addAll(repositoryRegistry.getRemoteRepositories());
 254         }
 255
 256         ModelBuildingRequest req =
 257                 new DefaultModelBuildingRequest().setProcessPlugins(false).setPomFile(file.toFile()).setTwoPhaseBuilding(
 258                         false).setValidationLevel(ModelBuildingRequest.VALIDATION_LEVEL_MINIMAL);
 259
 260         //MRM-1607. olamy this will resolve jdk profiles on the current running archiva jvm
 261         req.setSystemProperties(System.getProperties());
 262
 263         // MRM-1411
 264         req.setModelResolver(
 265                 new RepositoryModelResolver(managedRepository, pathTranslator, wagonFactory, remoteRepositories,
 266                         networkProxies, managedRepository, mavenSystemManager));
 267
 268         Model model;
 269         try {
 270             model = builder.build(req).getEffectiveModel();
 271         } catch (ModelBuildingException e) {
 272             String msg = "The artifact's POM file '" + file + "' was invalid: " + e.getMessage();
 273
 274             List<ModelProblem> modelProblems = e.getProblems();
 275             for (ModelProblem problem : modelProblems) {
 276                 // MRM-1411, related to MRM-1335
 277                 // this means that the problem was that the parent wasn't resolved!
 278                 // olamy really hackhish but fail with java profile so use error message
 279                 // || ( StringUtils.startsWith( problem.getMessage(), "Failed to determine Java version for profile" ) )
 280                 // but setTwoPhaseBuilding(true) fix that
 281                 if (((problem.getException() instanceof FileNotFoundException
 282                         || problem.getException() instanceof NoSuchFileException
 283                 ) && e.getModelId() != null &&
 284                         !e.getModelId().equals(problem.getModelId()))) {
 285                     LOGGER.warn("The artifact's parent POM file '{}' cannot be resolved. "
 286                             + "Using defaults for project version metadata..", file);
 287
 288                     ProjectVersionMetadata metadata = new ProjectVersionMetadata();
 289                     metadata.setId(readMetadataRequest.getProjectVersion());
 290
 291                     MavenProjectFacet facet = new MavenProjectFacet();
 292                     facet.setGroupId(readMetadataRequest.getNamespace());
 293                     facet.setArtifactId(readMetadataRequest.getProjectId());
 294                     facet.setPackaging("jar");
 295                     metadata.addFacet(facet);
 296
 297                     String errMsg =
 298                             "Error in resolving artifact's parent POM file. " + (problem.getException() == null
 299                                     ? problem.getMessage()
 300                                     : problem.getException().getMessage());
 301                     RepositoryProblemFacet repoProblemFacet = new RepositoryProblemFacet();
 302                     repoProblemFacet.setRepositoryId(readMetadataRequest.getRepositoryId());
 303                     repoProblemFacet.setId(readMetadataRequest.getRepositoryId());
 304                     repoProblemFacet.setMessage(errMsg);
 305                     repoProblemFacet.setProblem(errMsg);
 306                     repoProblemFacet.setProject(readMetadataRequest.getProjectId());
 307                     repoProblemFacet.setVersion(readMetadataRequest.getProjectVersion());
 308                     repoProblemFacet.setNamespace(readMetadataRequest.getNamespace());
 309
 310                     metadata.addFacet(repoProblemFacet);
 311
 312                     return metadata;
 313                 }
 314             }
 315
 316             throw new RepositoryStorageMetadataInvalidException("invalid-pom", msg, e);
 317         }
 318
 319         // Check if the POM is in the correct location
 320         boolean correctGroupId = readMetadataRequest.getNamespace().equals(model.getGroupId());
 321         boolean correctArtifactId = readMetadataRequest.getProjectId().equals(model.getArtifactId());
 322         boolean correctVersion = readMetadataRequest.getProjectVersion().equals(model.getVersion());
 323         if (!correctGroupId || !correctArtifactId || !correctVersion) {
 324             StringBuilder message = new StringBuilder("Incorrect POM coordinates in '" + file + "':");
 325             if (!correctGroupId) {
 326                 message.append("\nIncorrect group ID: ").append(model.getGroupId());
 327             }
 328             if (!correctArtifactId) {
 329                 message.append("\nIncorrect artifact ID: ").append(model.getArtifactId());
 330             }
 331             if (!correctVersion) {
 332                 message.append("\nIncorrect version: ").append(model.getVersion());
 333             }
 334
 335             throw new RepositoryStorageMetadataInvalidException("mislocated-pom", message.toString());
 336         }
 337
 338         ProjectVersionMetadata metadata = new ProjectVersionMetadata();
 339         metadata.setCiManagement(convertCiManagement(model.getCiManagement()));
 340         metadata.setDescription(model.getDescription());
 341         metadata.setId(readMetadataRequest.getProjectVersion());
 342         metadata.setIssueManagement(convertIssueManagement(model.getIssueManagement()));
 343         metadata.setLicenses(convertLicenses(model.getLicenses()));
 344         metadata.setMailingLists(convertMailingLists(model.getMailingLists()));
 345         metadata.setDependencies(convertDependencies(model.getDependencies()));
 346         metadata.setName(model.getName());
 347         metadata.setOrganization(convertOrganization(model.getOrganization()));
 348         metadata.setScm(convertScm(model.getScm()));
 349         metadata.setUrl(model.getUrl());
 350         metadata.setProperties(model.getProperties());
 351
 352         MavenProjectFacet facet = new MavenProjectFacet();
 353         facet.setGroupId(model.getGroupId() != null ? model.getGroupId() : model.getParent().getGroupId());
 354         facet.setArtifactId(model.getArtifactId());
 355         facet.setPackaging(model.getPackaging());
 356         if (model.getParent() != null) {
 357             MavenProjectParent parent = new MavenProjectParent();
 358             parent.setGroupId(model.getParent().getGroupId());
 359             parent.setArtifactId(model.getParent().getArtifactId());
 360             parent.setVersion(model.getParent().getVersion());
 361             facet.setParent(parent);
 362         }
 363         metadata.addFacet(facet);
 364
 365         return metadata;
 366
 367
 368     }
 369
 370     public void setWagonFactory(WagonFactory wagonFactory) {
 371         this.wagonFactory = wagonFactory;
 372     }
 373
 374     private List<org.apache.archiva.metadata.model.Dependency> convertDependencies(List<Dependency> dependencies) {
 375         List<org.apache.archiva.metadata.model.Dependency> l = new ArrayList<>();
 376         for (Dependency dependency : dependencies) {
 377             org.apache.archiva.metadata.model.Dependency newDependency =
 378                     new org.apache.archiva.metadata.model.Dependency();
 379             newDependency.setArtifactId(dependency.getArtifactId());
 380             newDependency.setClassifier(dependency.getClassifier());
 381             newDependency.setGroupId(dependency.getGroupId());
 382             newDependency.setOptional(dependency.isOptional());
 383             newDependency.setScope(dependency.getScope());
 384             newDependency.setSystemPath(dependency.getSystemPath());
 385             newDependency.setType(dependency.getType());
 386             newDependency.setVersion(dependency.getVersion());
 387             l.add(newDependency);
 388         }
 389         return l;
 390     }
 391
 392     private org.apache.archiva.metadata.model.Scm convertScm(Scm scm) {
 393         org.apache.archiva.metadata.model.Scm newScm = null;
 394         if (scm != null) {
 395             newScm = new org.apache.archiva.metadata.model.Scm();
 396             newScm.setConnection(scm.getConnection());
 397             newScm.setDeveloperConnection(scm.getDeveloperConnection());
 398             newScm.setUrl(scm.getUrl());
 399         }
 400         return newScm;
 401     }
 402
 403     private org.apache.archiva.metadata.model.Organization convertOrganization(Organization organization) {
 404         org.apache.archiva.metadata.model.Organization org = null;
 405         if (organization != null) {
 406             org = new org.apache.archiva.metadata.model.Organization();
 407             org.setName(organization.getName());
 408             org.setUrl(organization.getUrl());
 409         }
 410         return org;
 411     }
 412
 413     private List<org.apache.archiva.metadata.model.License> convertLicenses(List<License> licenses) {
 414         List<org.apache.archiva.metadata.model.License> l = new ArrayList<>();
 415         for (License license : licenses) {
 416             org.apache.archiva.metadata.model.License newLicense = new org.apache.archiva.metadata.model.License();
 417             newLicense.setName(license.getName());
 418             newLicense.setUrl(license.getUrl());
 419             l.add(newLicense);
 420         }
 421         return l;
 422     }
 423
 424     private List<org.apache.archiva.metadata.model.MailingList> convertMailingLists(List<MailingList> mailingLists) {
 425         List<org.apache.archiva.metadata.model.MailingList> l = new ArrayList<>();
 426         for (MailingList mailingList : mailingLists) {
 427             org.apache.archiva.metadata.model.MailingList newMailingList =
 428                     new org.apache.archiva.metadata.model.MailingList();
 429             newMailingList.setName(mailingList.getName());
 430             newMailingList.setMainArchiveUrl(mailingList.getArchive());
 431             newMailingList.setPostAddress(mailingList.getPost());
 432             newMailingList.setSubscribeAddress(mailingList.getSubscribe());
 433             newMailingList.setUnsubscribeAddress(mailingList.getUnsubscribe());
 434             newMailingList.setOtherArchives(mailingList.getOtherArchives());
 435             l.add(newMailingList);
 436         }
 437         return l;
 438     }
 439
 440     private org.apache.archiva.metadata.model.IssueManagement convertIssueManagement(IssueManagement issueManagement) {
 441         org.apache.archiva.metadata.model.IssueManagement im = null;
 442         if (issueManagement != null) {
 443             im = new org.apache.archiva.metadata.model.IssueManagement();
 444             im.setSystem(issueManagement.getSystem());
 445             im.setUrl(issueManagement.getUrl());
 446         }
 447         return im;
 448     }
 449
 450     private org.apache.archiva.metadata.model.CiManagement convertCiManagement(CiManagement ciManagement) {
 451         org.apache.archiva.metadata.model.CiManagement ci = null;
 452         if (ciManagement != null) {
 453             ci = new org.apache.archiva.metadata.model.CiManagement();
 454             ci.setSystem(ciManagement.getSystem());
 455             ci.setUrl(ciManagement.getUrl());
 456         }
 457         return ci;
 458     }
 459
 460     @Override
 461     public Collection<String> listRootNamespaces(String repoId, Filter<String> filter)
 462             throws RepositoryStorageRuntimeException {
 463         Path dir = getRepositoryBasedir(repoId);
 464
 465         return getSortedFiles(dir, filter);
 466     }
 467
 468     private static Collection<String> getSortedFiles(Path dir, Filter<String> filter) {
 469
 470         try (Stream<Path> stream = Files.list(dir)) {
 471             final Predicate<Path> dFilter = new DirectoryFilter(filter);
 472             return stream.filter(Files::isDirectory)
 473                     .filter(dFilter)
 474                     .map(path -> path.getFileName().toString())
 475                     .sorted().collect(Collectors.toList());
 476
 477         } catch (IOException e) {
 478             LOGGER.error("Could not read directory list {}: {}", dir, e.getMessage(), e);
 479             return Collections.emptyList();
 480         }
 481     }
 482
 483     private Path getRepositoryBasedir(String repoId)
 484             throws RepositoryStorageRuntimeException {
 485         ManagedRepository repositoryConfiguration = repositoryRegistry.getManagedRepository(repoId);
 486
 487         return Paths.get(repositoryConfiguration.getLocation());
 488     }
 489
 490     @Override
 491     public Collection<String> listNamespaces(String repoId, String namespace, Filter<String> filter)
 492             throws RepositoryStorageRuntimeException {
 493         Path dir = pathTranslator.toFile(getRepositoryBasedir(repoId), namespace);
 494         if (!(Files.exists(dir) && Files.isDirectory(dir))) {
 495             return Collections.emptyList();
 496         }
 497         // scan all the directories which are potential namespaces. Any directories known to be projects are excluded
 498         Predicate<Path> dFilter = new DirectoryFilter(filter);
 499         try (Stream<Path> stream = Files.list(dir)) {
 500             return stream.filter(dFilter).filter(path -> !isProject(path, filter)).map(path -> path.getFileName().toString())
 501                     .sorted().collect(Collectors.toList());
 502         } catch (IOException e) {
 503             LOGGER.error("Could not read directory {}: {}", dir, e.getMessage(), e);
 504             return Collections.emptyList();
 505         }
 506     }
 507
 508     @Override
 509     public Collection<String> listProjects(String repoId, String namespace, Filter<String> filter)
 510             throws RepositoryStorageRuntimeException {
 511         Path dir = pathTranslator.toFile(getRepositoryBasedir(repoId), namespace);
 512         if (!(Files.exists(dir) && Files.isDirectory(dir))) {
 513             return Collections.emptyList();
 514         }
 515         // scan all directories in the namespace, and only include those that are known to be projects
 516         final Predicate<Path> dFilter = new DirectoryFilter(filter);
 517         try (Stream<Path> stream = Files.list(dir)) {
 518             return stream.filter(dFilter).filter(path -> isProject(path, filter)).map(path -> path.getFileName().toString())
 519                     .sorted().collect(Collectors.toList());
 520         } catch (IOException e) {
 521             LOGGER.error("Could not read directory {}: {}", dir, e.getMessage(), e);
 522             return Collections.emptyList();
 523         }
 524
 525     }
 526
 527     @Override
 528     public Collection<String> listProjectVersions(String repoId, String namespace, String projectId,
 529                                                   Filter<String> filter)
 530             throws RepositoryStorageRuntimeException {
 531         Path dir = pathTranslator.toFile(getRepositoryBasedir(repoId), namespace, projectId);
 532         if (!(Files.exists(dir) && Files.isDirectory(dir))) {
 533             return Collections.emptyList();
 534         }
 535
 536         // all directories in a project directory can be considered a version
 537         return getSortedFiles(dir, filter);
 538     }
 539
 540     @Override
 541     public Collection<ArtifactMetadata> readArtifactsMetadata(ReadMetadataRequest readMetadataRequest)
 542             throws RepositoryStorageRuntimeException {
 543         Path dir = pathTranslator.toFile(getRepositoryBasedir(readMetadataRequest.getRepositoryId()),
 544                 readMetadataRequest.getNamespace(), readMetadataRequest.getProjectId(),
 545                 readMetadataRequest.getProjectVersion());
 546         if (!(Files.exists(dir) && Files.isDirectory(dir))) {
 547             return Collections.emptyList();
 548         }
 549
 550         // all files that are not metadata and not a checksum / signature are considered artifacts
 551         final Predicate<Path> dFilter = new ArtifactDirectoryFilter(readMetadataRequest.getFilter());
 552         try (Stream<Path> stream = Files.list(dir)) {
 553             // Returns a map TRUE -> (success values), FALSE -> (Exceptions)
 554             Map<Boolean, List<Try<ArtifactMetadata>>> result = stream.filter(dFilter).map(path -> {
 555                         try {
 556                             return Try.success(getArtifactFromFile(readMetadataRequest.getRepositoryId(), readMetadataRequest.getNamespace(),
 557                                     readMetadataRequest.getProjectId(), readMetadataRequest.getProjectVersion(),
 558                                     path));
 559                         } catch (Exception e) {
 560                             LOGGER.debug("Could not create metadata for {}:  {}", path, e.getMessage(), e);
 561                             return Try.<ArtifactMetadata>failure(e);
 562                         }
 563                     }
 564             ).collect(Collectors.groupingBy(Try::isSuccess));
 565             if (result.containsKey(Boolean.FALSE) && result.get(Boolean.FALSE).size() > 0 && (!result.containsKey(Boolean.TRUE) || result.get(Boolean.TRUE).size() == 0)) {
 566                 LOGGER.error("Could not get artifact metadata. Directory: {}. Number of errors {}.", dir, result.get(Boolean.FALSE).size());
 567                 Try<ArtifactMetadata> failure = result.get(Boolean.FALSE).get(0);
 568                 LOGGER.error("Sample exception {}", failure.getError().getMessage(), failure.getError());
 569                 throw new RepositoryStorageRuntimeException(readMetadataRequest.getRepositoryId(), "Could not retrieve metadata of the files");
 570             } else {
 571                 if (!result.containsKey(Boolean.TRUE) || result.get(Boolean.TRUE) == null) {
 572                     return Collections.emptyList();
 573                 }
 574                 return result.get(Boolean.TRUE).stream().map(tr -> tr.get()).collect(Collectors.toList());
 575             }
 576         } catch (IOException e) {
 577             LOGGER.error("Could not read directory {}: {}", dir, e.getMessage(), e);
 578         }
 579         return Collections.emptyList();
 580
 581     }
 582
 583     @Override
 584     public ArtifactMetadata readArtifactMetadataFromPath(String repoId, String path)
 585             throws RepositoryStorageRuntimeException {
 586         ArtifactMetadata metadata = pathTranslator.getArtifactForPath(repoId, path);
 587
 588         try {
 589             populateArtifactMetadataFromFile(metadata, getRepositoryBasedir(repoId).resolve(path));
 590         } catch (IOException e) {
 591             throw new RepositoryStorageRuntimeException(repoId, "Error during metadata retrieval of " + path + " :" + e.getMessage(), e);
 592         }
 593
 594         return metadata;
 595     }
 596
 597     private ArtifactMetadata getArtifactFromFile(String repoId, String namespace, String projectId,
 598                                                  String projectVersion, Path file) throws IOException {
 599         ArtifactMetadata metadata =
 600                 pathTranslator.getArtifactFromId(repoId, namespace, projectId, projectVersion, file.getFileName().toString());
 601
 602         populateArtifactMetadataFromFile(metadata, file);
 603
 604         return metadata;
 605     }
 606
 607     @Override
 608     public void applyServerSideRelocation(ManagedRepositoryContent managedRepository, ArtifactReference artifact)
 609             throws ProxyDownloadException {
 610         if ("pom".equals(artifact.getType())) {
 611             return;
 612         }
 613
 614         // Build the artifact POM reference
 615         ArtifactReference pomReference = new ArtifactReference();
 616         pomReference.setGroupId(artifact.getGroupId());
 617         pomReference.setArtifactId(artifact.getArtifactId());
 618         pomReference.setVersion(artifact.getVersion());
 619         pomReference.setType("pom");
 620
 621         RepositoryType repositoryType = managedRepository.getRepository().getType();
 622         if (!proxyRegistry.hasHandler(repositoryType)) {
 623             throw new ProxyDownloadException("No proxy handler found for repository type "+repositoryType, new HashMap<>());
 624         }
 625
 626         RepositoryProxyHandler proxyHandler = proxyRegistry.getHandler(repositoryType).get(0);
 627
 628         // Get the artifact POM from proxied repositories if needed
 629         proxyHandler.fetchFromProxies(managedRepository, pomReference);
 630
 631         // Open and read the POM from the managed repo
 632         Path pom = managedRepository.toFile(pomReference);
 633
 634         if (!Files.exists(pom)) {
 635             return;
 636         }
 637
 638         try {
 639             // MavenXpp3Reader leaves the file open, so we need to close it ourselves.
 640
 641             Model model = null;
 642             try (Reader reader = Files.newBufferedReader(pom, Charset.defaultCharset())) {
 643                 model = MAVEN_XPP_3_READER.read(reader);
 644             }
 645
 646             DistributionManagement dist = model.getDistributionManagement();
 647             if (dist != null) {
 648                 Relocation relocation = dist.getRelocation();
 649                 if (relocation != null) {
 650                     // artifact is relocated : update the repositoryPath
 651                     if (relocation.getGroupId() != null) {
 652                         artifact.setGroupId(relocation.getGroupId());
 653                     }
 654                     if (relocation.getArtifactId() != null) {
 655                         artifact.setArtifactId(relocation.getArtifactId());
 656                     }
 657                     if (relocation.getVersion() != null) {
 658                         artifact.setVersion(relocation.getVersion());
 659                     }
 660                 }
 661             }
 662         } catch (IOException e) {
 663             // Unable to read POM : ignore.
 664         } catch (XmlPullParserException e) {
 665             // Invalid POM : ignore
 666         }
 667     }
 668
 669
 670     @Override
 671     public String getFilePath(String requestPath, org.apache.archiva.repository.ManagedRepository managedRepository) {
 672         // managedRepository can be null
 673         // extract artifact reference from url
 674         // groupId:artifactId:version:packaging:classifier
 675         //org/apache/archiva/archiva-checksum/1.4-M4-SNAPSHOT/archiva-checksum-1.4-M4-SNAPSHOT.jar
 676         String logicalResource = null;
 677         String requestPathInfo = StringUtils.defaultString(requestPath);
 678
 679         //remove prefix ie /repository/blah becomes /blah
 680         requestPathInfo = removePrefix(requestPathInfo);
 681
 682         // Remove prefixing slash as the repository id doesn't contain it;
 683         if (requestPathInfo.startsWith("/")) {
 684             requestPathInfo = requestPathInfo.substring(1);
 685         }
 686
 687         int slash = requestPathInfo.indexOf('/');
 688         if (slash > 0) {
 689             logicalResource = requestPathInfo.substring(slash);
 690
 691             if (logicalResource.endsWith("/..")) {
 692                 logicalResource += "/";
 693             }
 694
 695             if (logicalResource != null && logicalResource.startsWith("//")) {
 696                 logicalResource = logicalResource.substring(1);
 697             }
 698
 699             if (logicalResource == null) {
 700                 logicalResource = "/";
 701             }
 702         } else {
 703             logicalResource = "/";
 704         }
 705         return logicalResource;
 706
 707     }
 708
 709     @Override
 710     public String getFilePathWithVersion(final String requestPath, ManagedRepositoryContent managedRepositoryContent)
 711             throws XMLException, RelocationException {
 712
 713         if (StringUtils.endsWith(requestPath, METADATA_FILENAME)) {
 714             return getFilePath(requestPath, managedRepositoryContent.getRepository());
 715         }
 716
 717         String filePath = getFilePath(requestPath, managedRepositoryContent.getRepository());
 718
 719         ArtifactReference artifactReference = null;
 720         try {
 721             artifactReference = pathParser.toArtifactReference(filePath);
 722         } catch (LayoutException e) {
 723             return filePath;
 724         }
 725
 726         if (StringUtils.endsWith(artifactReference.getVersion(), VersionUtil.SNAPSHOT)) {
 727             // read maven metadata to get last timestamp
 728             Path metadataDir = Paths.get(managedRepositoryContent.getRepoRoot(), filePath).getParent();
 729             if (!Files.exists(metadataDir)) {
 730                 return filePath;
 731             }
 732             Path metadataFile = metadataDir.resolve(METADATA_FILENAME);
 733             if (!Files.exists(metadataFile)) {
 734                 return filePath;
 735             }
 736             ArchivaRepositoryMetadata archivaRepositoryMetadata = MavenMetadataReader.read(metadataFile);
 737             int buildNumber = archivaRepositoryMetadata.getSnapshotVersion().getBuildNumber();
 738             String timestamp = archivaRepositoryMetadata.getSnapshotVersion().getTimestamp();
 739
 740             // MRM-1846
 741             if (buildNumber < 1 && timestamp == null) {
 742                 return filePath;
 743             }
 744
 745             // org/apache/archiva/archiva-checksum/1.4-M4-SNAPSHOT/archiva-checksum-1.4-M4-SNAPSHOT.jar
 746             // ->  archiva-checksum-1.4-M4-20130425.081822-1.jar
 747
 748             filePath = StringUtils.replace(filePath, //
 749                     artifactReference.getArtifactId() //
 750                             + "-" + artifactReference.getVersion(), //
 751                     artifactReference.getArtifactId() //
 752                             + "-" + StringUtils.remove(artifactReference.getVersion(),
 753                             "-" + VersionUtil.SNAPSHOT) //
 754                             + "-" + timestamp //
 755                             + "-" + buildNumber);
 756
 757             throw new RelocationException("/repository/" + managedRepositoryContent.getRepository().getId() +
 758                     (StringUtils.startsWith(filePath, "/") ? "" : "/") + filePath,
 759                     RelocationException.RelocationType.TEMPORARY);
 760
 761         }
 762
 763         return filePath;
 764     }
 765
 766     //-----------------------------
 767     // internal
 768     //-----------------------------
 769
 770     /**
 771      * FIXME remove
 772      *
 773      * @param href
 774      * @return
 775      */
 776     private static String removePrefix(final String href) {
 777         String[] parts = StringUtils.split(href, '/');
 778         parts = (String[]) ArrayUtils.subarray(parts, 1, parts.length);
 779         if (parts == null || parts.length == 0) {
 780             return "/";
 781         }
 782
 783         String joinedString = StringUtils.join(parts, '/');
 784         if (href.endsWith("/")) {
 785             joinedString = joinedString + "/";
 786         }
 787
 788         return joinedString;
 789     }
 790
 791     private static void populateArtifactMetadataFromFile(ArtifactMetadata metadata, Path file) throws IOException {
 792         metadata.setWhenGathered(new Date());
 793         metadata.setFileLastModified(Files.getLastModifiedTime(file).toMillis());
 794         ChecksummedFile checksummedFile = new ChecksummedFile(file);
 795         try {
 796             metadata.setMd5(checksummedFile.calculateChecksum(ChecksumAlgorithm.MD5));
 797         } catch (IOException e) {
 798             LOGGER.error("Unable to checksum file {}: {},MD5", file, e.getMessage());
 799         }
 800         try {
 801             metadata.setSha1(checksummedFile.calculateChecksum(ChecksumAlgorithm.SHA1));
 802         } catch (IOException e) {
 803             LOGGER.error("Unable to checksum file {}: {},SHA1", file, e.getMessage());
 804         }
 805         metadata.setSize(Files.size(file));
 806     }
 807
 808     private boolean isProject(Path dir, Filter<String> filter) {
 809         // scan directories for a valid project version subdirectory, meaning this must be a project directory
 810         final Predicate<Path> dFilter = new DirectoryFilter(filter);
 811         try (Stream<Path> stream = Files.list(dir)) {
 812             boolean projFound = stream.filter(dFilter)
 813                     .anyMatch(path -> isProjectVersion(path));
 814             if (projFound) {
 815                 return true;
 816             }
 817         } catch (IOException e) {
 818             LOGGER.error("Could not read directory list {}: {}", dir, e.getMessage(), e);
 819         }
 820
 821         // if a metadata file is present, check if this is the "artifactId" directory, marking it as a project
 822         ArchivaRepositoryMetadata metadata = readMetadata(dir);
 823         if (metadata != null && dir.getFileName().toString().equals(metadata.getArtifactId())) {
 824             return true;
 825         }
 826
 827         return false;
 828     }
 829
 830     private boolean isProjectVersion(Path dir) {
 831         final String artifactId = dir.getParent().getFileName().toString();
 832         final String projectVersion = dir.getFileName().toString();
 833
 834         // check if there is a POM artifact file to ensure it is a version directory
 835
 836         Predicate<Path> filter;
 837         if (VersionUtil.isSnapshot(projectVersion)) {
 838             filter = new PomFilenameFilter(artifactId, projectVersion);
 839         } else {
 840             final String pomFile = artifactId + "-" + projectVersion + ".pom";
 841             filter = new PomFileFilter(pomFile);
 842         }
 843         try (Stream<Path> stream = Files.list(dir)) {
 844             if (stream.filter(Files::isRegularFile).anyMatch(filter)) {
 845                 return true;
 846             }
 847         } catch (IOException e) {
 848             LOGGER.error("Could not list directory {}: {}", dir, e.getMessage(), e);
 849         }
 850
 851         // if a metadata file is present, check if this is the "version" directory, marking it as a project version
 852         ArchivaRepositoryMetadata metadata = readMetadata(dir);
 853         if (metadata != null && projectVersion.equals(metadata.getVersion())) {
 854             return true;
 855         }
 856
 857         return false;
 858     }
 859
 860     private ArchivaRepositoryMetadata readMetadata(Path directory) {
 861         ArchivaRepositoryMetadata metadata = null;
 862         Path metadataFile = directory.resolve(METADATA_FILENAME);
 863         if (Files.exists(metadataFile)) {
 864             try {
 865                 metadata = MavenMetadataReader.read(metadataFile);
 866             } catch (XMLException e) {
 867                 // ignore missing or invalid metadata
 868             }
 869         }
 870         return metadata;
 871     }
 872
 873     private static class DirectoryFilter
 874             implements Predicate<Path> {
 875         private final Filter<String> filter;
 876
 877         public DirectoryFilter(Filter<String> filter) {
 878             this.filter = filter;
 879         }
 880
 881         @Override
 882         public boolean test(Path dir) {
 883             final String name = dir.getFileName().toString();
 884             if (!filter.accept(name)) {
 885                 return false;
 886             } else if (name.startsWith(".")) {
 887                 return false;
 888             } else if (!Files.isDirectory(dir)) {
 889                 return false;
 890             }
 891             return true;
 892         }
 893     }
 894
 895     private static class ArtifactDirectoryFilter
 896             implements Predicate<Path> {
 897         private final Filter<String> filter;
 898
 899         private ArtifactDirectoryFilter(Filter<String> filter) {
 900             this.filter = filter;
 901         }
 902
 903         @Override
 904         public boolean test(Path dir) {
 905             final String name = dir.getFileName().toString();
 906             // TODO compare to logic in maven-repository-layer
 907             if (!filter.accept(name)) {
 908                 return false;
 909             } else if (name.startsWith(".")) {
 910                 return false;
 911             } else if (name.endsWith(".md5") || name.endsWith(".sha1") || name.endsWith(".asc")) {
 912                 return false;
 913             } else if (Arrays.binarySearch(IGNORED_FILES, name) >= 0) {
 914                 return false;
 915             } else if (Files.isDirectory(dir)) {
 916                 return false;
 917             }
 918             // some files from remote repositories can have name like maven-metadata-archiva-vm-all-public.xml
 919             else if (StringUtils.startsWith(name, METADATA_FILENAME_START) && StringUtils.endsWith(name, ".xml")) {
 920                 return false;
 921             }
 922
 923             return true;
 924
 925         }
 926     }
 927
 928
 929     private static final class PomFilenameFilter
 930             implements Predicate<Path> {
 931
 932         private final String artifactId, projectVersion;
 933
 934         private PomFilenameFilter(String artifactId, String projectVersion) {
 935             this.artifactId = artifactId;
 936             this.projectVersion = projectVersion;
 937         }
 938
 939         @Override
 940         public boolean test(Path dir) {
 941             final String name = dir.getFileName().toString();
 942             if (name.startsWith(artifactId + "-") && name.endsWith(".pom")) {
 943                 String v = name.substring(artifactId.length() + 1, name.length() - 4);
 944                 v = VersionUtil.getBaseVersion(v);
 945                 if (v.equals(projectVersion)) {
 946                     return true;
 947                 }
 948             }
 949             return false;
 950         }
 951
 952     }
 953
 954     private static class PomFileFilter
 955             implements Predicate<Path> {
 956         private final String pomFile;
 957
 958         private PomFileFilter(String pomFile) {
 959             this.pomFile = pomFile;
 960         }
 961
 962         @Override
 963         public boolean test(Path dir) {
 964             return pomFile.equals(dir.getFileName().toString());
 965         }
 966     }
 967
 968
 969     public PathParser getPathParser() {
 970         return pathParser;
 971     }
 972
 973     public void setPathParser(PathParser pathParser) {
 974         this.pathParser = pathParser;
 975     }
 976 }