source.dussan.org Git - archiva.git/blob

   1 package org.apache.archiva.repository.maven.metadata.storage;
   2
   3 /*
   4  * Licensed to the Apache Software Foundation (ASF) under one
   5  * or more contributor license agreements.  See the NOTICE file
   6  * distributed with this work for additional information
   7  * regarding copyright ownership.  The ASF licenses this file
   8  * to you under the Apache License, Version 2.0 (the
   9  * "License"); you may not use this file except in compliance
  10  * with the License.  You may obtain a copy of the License at
  11  *
  12  * http://www.apache.org/licenses/LICENSE-2.0
  13  * Unless required by applicable law or agreed to in writing,
  14  * software distributed under the License is distributed on an
  15  * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
  16  * KIND, either express or implied.  See the License for the
  17  * specific language governing permissions and limitations
  18  * under the License.
  19  */
  20
  21 import org.apache.archiva.checksum.ChecksumAlgorithm;
  22 import org.apache.archiva.checksum.ChecksummedFile;
  23 import org.apache.archiva.common.Try;
  24 import org.apache.archiva.common.utils.VersionUtil;
  25 import org.apache.archiva.filter.Filter;
  26 import org.apache.archiva.metadata.maven.MavenMetadataReader;
  27 import org.apache.archiva.metadata.model.ArtifactMetadata;
  28 import org.apache.archiva.metadata.model.ProjectMetadata;
  29 import org.apache.archiva.metadata.model.ProjectVersionMetadata;
  30 import org.apache.archiva.metadata.model.facets.RepositoryProblemFacet;
  31 import org.apache.archiva.metadata.repository.storage.ReadMetadataRequest;
  32 import org.apache.archiva.metadata.repository.storage.RelocationException;
  33 import org.apache.archiva.metadata.repository.storage.RepositoryPathTranslator;
  34 import org.apache.archiva.metadata.repository.storage.RepositoryStorage;
  35 import org.apache.archiva.metadata.repository.storage.RepositoryStorageMetadataInvalidException;
  36 import org.apache.archiva.metadata.repository.storage.RepositoryStorageMetadataNotFoundException;
  37 import org.apache.archiva.metadata.repository.storage.RepositoryStorageRuntimeException;
  38 import org.apache.archiva.model.ArchivaRepositoryMetadata;
  39 import org.apache.archiva.model.SnapshotVersion;
  40 import org.apache.archiva.policies.ProxyDownloadException;
  41 import org.apache.archiva.proxy.ProxyRegistry;
  42 import org.apache.archiva.proxy.maven.WagonFactory;
  43 import org.apache.archiva.proxy.model.NetworkProxy;
  44 import org.apache.archiva.proxy.model.ProxyConnector;
  45 import org.apache.archiva.proxy.model.RepositoryProxyHandler;
  46 import org.apache.archiva.repository.ManagedRepository;
  47 import org.apache.archiva.repository.ManagedRepositoryContent;
  48 import org.apache.archiva.repository.ReleaseScheme;
  49 import org.apache.archiva.repository.RemoteRepository;
  50 import org.apache.archiva.repository.RepositoryRegistry;
  51 import org.apache.archiva.repository.RepositoryType;
  52 import org.apache.archiva.repository.content.Artifact;
  53 import org.apache.archiva.repository.content.BaseRepositoryContentLayout;
  54 import org.apache.archiva.repository.content.ContentItem;
  55 import org.apache.archiva.repository.content.ItemSelector;
  56 import org.apache.archiva.repository.content.LayoutException;
  57 import org.apache.archiva.repository.content.base.ArchivaItemSelector;
  58 import org.apache.archiva.repository.maven.MavenSystemManager;
  59 import org.apache.archiva.repository.metadata.RepositoryMetadataException;
  60 import org.apache.archiva.repository.storage.StorageAsset;
  61 import org.apache.commons.lang3.ArrayUtils;
  62 import org.apache.commons.lang3.StringUtils;
  63 import org.apache.maven.model.CiManagement;
  64 import org.apache.maven.model.Dependency;
  65 import org.apache.maven.model.DistributionManagement;
  66 import org.apache.maven.model.IssueManagement;
  67 import org.apache.maven.model.License;
  68 import org.apache.maven.model.MailingList;
  69 import org.apache.maven.model.Model;
  70 import org.apache.maven.model.Organization;
  71 import org.apache.maven.model.Relocation;
  72 import org.apache.maven.model.Scm;
  73 import org.apache.maven.model.building.DefaultModelBuilderFactory;
  74 import org.apache.maven.model.building.DefaultModelBuildingRequest;
  75 import org.apache.maven.model.building.ModelBuilder;
  76 import org.apache.maven.model.building.ModelBuildingException;
  77 import org.apache.maven.model.building.ModelBuildingRequest;
  78 import org.apache.maven.model.building.ModelProblem;
  79 import org.apache.maven.model.io.xpp3.MavenXpp3Reader;
  80 import org.codehaus.plexus.util.xml.pull.XmlPullParserException;
  81 import org.slf4j.Logger;
  82 import org.slf4j.LoggerFactory;
  83 import org.springframework.context.ApplicationContext;
  84 import org.springframework.stereotype.Service;
  85
  86 import javax.annotation.PostConstruct;
  87 import javax.inject.Inject;
  88 import javax.inject.Named;
  89 import java.io.FileNotFoundException;
  90 import java.io.IOException;
  91 import java.io.Reader;
  92 import java.nio.channels.Channels;
  93 import java.nio.charset.Charset;
  94 import java.nio.file.NoSuchFileException;
  95 import java.time.ZoneId;
  96 import java.time.ZonedDateTime;
  97 import java.util.ArrayList;
  98 import java.util.Arrays;
  99 import java.util.Collection;
 100 import java.util.Collections;
 101 import java.util.HashMap;
 102 import java.util.List;
 103 import java.util.Map;
 104 import java.util.Set;
 105 import java.util.function.Predicate;
 106 import java.util.stream.Collectors;
 107
 108 // import java.io.FileNotFoundException;
 109
 110 /**
 111  * <p>
 112  * Maven 2 repository format storage implementation. This class currently takes parameters to indicate the repository to
 113  * deal with rather than being instantiated per-repository.
 114  * FIXME: instantiate one per repository and allocate permanently from a factory (which can be obtained within the session).
 115  * </p>
 116  * <p>
 117  * The session is passed in as an argument to obtain any necessary resources, rather than the class being instantiated
 118  * within the session in the context of a single managed repository's resolution needs.
 119  * </p>
 120  */
 121 @Service("repositoryStorage#maven2")
 122 public class Maven2RepositoryStorage
 123         implements RepositoryStorage {
 124
 125     private static final Logger log = LoggerFactory.getLogger(Maven2RepositoryStorage.class);
 126
 127     private ModelBuilder builder;
 128
 129     @Inject
 130     RepositoryRegistry repositoryRegistry;
 131
 132     @Inject
 133     @Named( "metadataReader#maven" )
 134     MavenMetadataReader metadataReader;
 135
 136     @Inject
 137     @Named("repositoryPathTranslator#maven2")
 138     private RepositoryPathTranslator pathTranslator;
 139
 140     @Inject
 141     private WagonFactory wagonFactory;
 142
 143     @Inject
 144     private ApplicationContext applicationContext;
 145
 146     @Inject
 147     private ProxyRegistry proxyRegistry;
 148
 149     @Inject
 150     private MavenSystemManager mavenSystemManager;
 151
 152     private static final String METADATA_FILENAME_START = "maven-metadata";
 153
 154     private static final String METADATA_FILENAME = METADATA_FILENAME_START + ".xml";
 155
 156     // This array must be lexically sorted
 157     private static final String[] IGNORED_FILES = {METADATA_FILENAME, "resolver-status.properties"};
 158
 159     private static final MavenXpp3Reader MAVEN_XPP_3_READER = new MavenXpp3Reader();
 160
 161
 162     @PostConstruct
 163     public void initialize() {
 164         builder = new DefaultModelBuilderFactory().newInstance();
 165
 166     }
 167
 168     @Override
 169     public ProjectMetadata readProjectMetadata(String repoId, String namespace, String projectId) {
 170         // TODO: could natively implement the "shared model" concept from the browse action to avoid needing it there?
 171         return null;
 172     }
 173
 174     @Override
 175     public ProjectVersionMetadata readProjectVersionMetadata(ReadMetadataRequest readMetadataRequest)
 176             throws RepositoryStorageMetadataNotFoundException, RepositoryStorageMetadataInvalidException,
 177             RepositoryStorageRuntimeException {
 178
 179         ManagedRepository managedRepository = repositoryRegistry.getManagedRepository(readMetadataRequest.getRepositoryId());
 180         boolean isReleases = managedRepository.getActiveReleaseSchemes().contains(ReleaseScheme.RELEASE);
 181         boolean isSnapshots = managedRepository.getActiveReleaseSchemes().contains(ReleaseScheme.SNAPSHOT);
 182         String artifactVersion = readMetadataRequest.getProjectVersion();
 183         // olamy: in case of browsing via the ui we can mix repos (parent of a SNAPSHOT can come from release repo)
 184         if (!readMetadataRequest.isBrowsingRequest()) {
 185             if (VersionUtil.isSnapshot(artifactVersion)) {
 186                 // skygo trying to improve speed by honoring managed configuration MRM-1658
 187                 if (isReleases && !isSnapshots) {
 188                     throw new RepositoryStorageRuntimeException("lookforsnaponreleaseonly",
 189                             "managed repo is configured for release only");
 190                 }
 191             } else {
 192                 if (!isReleases && isSnapshots) {
 193                     throw new RepositoryStorageRuntimeException("lookforsreleaseonsneponly",
 194                             "managed repo is configured for snapshot only");
 195                 }
 196             }
 197         }
 198         StorageAsset basedir = managedRepository.getRoot();
 199         if (VersionUtil.isSnapshot(artifactVersion)) {
 200             StorageAsset metadataFile = pathTranslator.toFile(basedir, readMetadataRequest.getNamespace(),
 201                     readMetadataRequest.getProjectId(), artifactVersion,
 202                     METADATA_FILENAME);
 203             try {
 204                 ArchivaRepositoryMetadata metadata = metadataReader.read(metadataFile);
 205
 206                 // re-adjust to timestamp if present, otherwise retain the original -SNAPSHOT filename
 207                 SnapshotVersion snapshotVersion = metadata.getSnapshotVersion();
 208                 if (snapshotVersion != null) {
 209                     artifactVersion =
 210                             artifactVersion.substring(0, artifactVersion.length() - 8); // remove SNAPSHOT from end
 211                     artifactVersion =
 212                             artifactVersion + snapshotVersion.getTimestamp() + "-" + snapshotVersion.getBuildNumber();
 213                 }
 214             } catch ( RepositoryMetadataException e) {
 215                 // unable to parse metadata - LOGGER it, and continue with the version as the original SNAPSHOT version
 216                 log.warn("Invalid metadata: {} - {}", metadataFile, e.getMessage());
 217             }
 218         }
 219
 220         // TODO: won't work well with some other layouts, might need to convert artifact parts to ID by path translator
 221         String id = readMetadataRequest.getProjectId() + "-" + artifactVersion + ".pom";
 222         StorageAsset file =
 223                 pathTranslator.toFile(basedir, readMetadataRequest.getNamespace(), readMetadataRequest.getProjectId(),
 224                         readMetadataRequest.getProjectVersion(), id);
 225
 226         if (!file.exists()) {
 227             // metadata could not be resolved
 228             throw new RepositoryStorageMetadataNotFoundException(
 229                     "The artifact's POM file '" + file.getPath() + "' was missing");
 230         }
 231
 232         // TODO: this is a workaround until we can properly resolve using proxies as well - this doesn't cache
 233         //       anything locally!
 234         List<RemoteRepository> remoteRepositories = new ArrayList<>();
 235         Map<String, NetworkProxy> networkProxies = new HashMap<>();
 236
 237         Map<String, List<ProxyConnector>> proxyConnectorsMap = proxyRegistry.getProxyConnectorAsMap();
 238         List<ProxyConnector> proxyConnectors = proxyConnectorsMap.get(readMetadataRequest.getRepositoryId());
 239         if (proxyConnectors != null) {
 240             for (ProxyConnector proxyConnector : proxyConnectors) {
 241                 RemoteRepository remoteRepoConfig =
 242                         repositoryRegistry.getRemoteRepository(proxyConnector.getTargetRepository().getId());
 243
 244                 if (remoteRepoConfig != null) {
 245                     remoteRepositories.add(remoteRepoConfig);
 246
 247                     NetworkProxy networkProxyConfig =
 248                             proxyRegistry.getNetworkProxy(proxyConnector.getProxyId());
 249
 250                     if (networkProxyConfig != null) {
 251                         // key/value: remote repo ID/proxy info
 252                         networkProxies.put(proxyConnector.getTargetRepository().getId(), networkProxyConfig);
 253                     }
 254                 }
 255             }
 256         }
 257
 258         // That's a browsing request so we can a mix of SNAPSHOT and release artifacts (especially with snapshots which
 259         // can have released parent pom
 260         if (readMetadataRequest.isBrowsingRequest()) {
 261             remoteRepositories.addAll(repositoryRegistry.getRemoteRepositories());
 262         }
 263
 264         ModelBuildingRequest req =
 265                 new DefaultModelBuildingRequest().setProcessPlugins(false).setPomFile(file.getFilePath().toFile()).setTwoPhaseBuilding(
 266                         false).setValidationLevel(ModelBuildingRequest.VALIDATION_LEVEL_MINIMAL);
 267
 268         //MRM-1607. olamy this will resolve jdk profiles on the current running archiva jvm
 269         req.setSystemProperties(System.getProperties());
 270
 271         // MRM-1411
 272         req.setModelResolver(
 273                 new RepositoryModelResolver(managedRepository, pathTranslator, wagonFactory, remoteRepositories,
 274                         networkProxies, managedRepository, mavenSystemManager, metadataReader));
 275
 276         Model model;
 277         try {
 278             model = builder.build(req).getEffectiveModel();
 279         } catch (ModelBuildingException e) {
 280             String msg = "The artifact's POM file '" + file + "' was invalid: " + e.getMessage();
 281
 282             List<ModelProblem> modelProblems = e.getProblems();
 283             for (ModelProblem problem : modelProblems) {
 284                 // MRM-1411, related to MRM-1335
 285                 // this means that the problem was that the parent wasn't resolved!
 286                 // olamy really hackhish but fail with java profile so use error message
 287                 // || ( StringUtils.startsWith( problem.getMessage(), "Failed to determine Java version for profile" ) )
 288                 // but setTwoPhaseBuilding(true) fix that
 289                 if (((problem.getException() instanceof FileNotFoundException
 290                         || problem.getException() instanceof NoSuchFileException
 291                 ) && e.getModelId() != null &&
 292                         !e.getModelId().equals(problem.getModelId()))) {
 293                     log.warn("The artifact's parent POM file '{}' cannot be resolved. "
 294                             + "Using defaults for project version metadata..", file);
 295
 296                     ProjectVersionMetadata metadata = new ProjectVersionMetadata();
 297                     metadata.setId(readMetadataRequest.getProjectVersion());
 298
 299                     MavenProjectFacet facet = new MavenProjectFacet();
 300                     facet.setGroupId(readMetadataRequest.getNamespace());
 301                     facet.setArtifactId(readMetadataRequest.getProjectId());
 302                     facet.setPackaging("jar");
 303                     metadata.addFacet(facet);
 304
 305                     String errMsg =
 306                             "Error in resolving artifact's parent POM file. " + (problem.getException() == null
 307                                     ? problem.getMessage()
 308                                     : problem.getException().getMessage());
 309                     RepositoryProblemFacet repoProblemFacet = new RepositoryProblemFacet();
 310                     repoProblemFacet.setRepositoryId(readMetadataRequest.getRepositoryId());
 311                     repoProblemFacet.setId(readMetadataRequest.getRepositoryId());
 312                     repoProblemFacet.setMessage(errMsg);
 313                     repoProblemFacet.setProblem(errMsg);
 314                     repoProblemFacet.setProject(readMetadataRequest.getProjectId());
 315                     repoProblemFacet.setVersion(readMetadataRequest.getProjectVersion());
 316                     repoProblemFacet.setNamespace(readMetadataRequest.getNamespace());
 317
 318                     metadata.addFacet(repoProblemFacet);
 319
 320                     return metadata;
 321                 }
 322             }
 323
 324             throw new RepositoryStorageMetadataInvalidException("invalid-pom", msg, e);
 325         }
 326
 327         // Check if the POM is in the correct location
 328         boolean correctGroupId = readMetadataRequest.getNamespace().equals(model.getGroupId());
 329         boolean correctArtifactId = readMetadataRequest.getProjectId().equals(model.getArtifactId());
 330         boolean correctVersion = readMetadataRequest.getProjectVersion().equals(model.getVersion());
 331         if (!correctGroupId || !correctArtifactId || !correctVersion) {
 332             StringBuilder message = new StringBuilder("Incorrect POM coordinates in '" + file + "':");
 333             if (!correctGroupId) {
 334                 message.append("\nIncorrect group ID: ").append(model.getGroupId());
 335             }
 336             if (!correctArtifactId) {
 337                 message.append("\nIncorrect artifact ID: ").append(model.getArtifactId());
 338             }
 339             if (!correctVersion) {
 340                 message.append("\nIncorrect version: ").append(model.getVersion());
 341             }
 342
 343             throw new RepositoryStorageMetadataInvalidException("mislocated-pom", message.toString());
 344         }
 345
 346         ProjectVersionMetadata metadata = new ProjectVersionMetadata();
 347         metadata.setCiManagement(convertCiManagement(model.getCiManagement()));
 348         metadata.setDescription(model.getDescription());
 349         metadata.setId(readMetadataRequest.getProjectVersion());
 350         metadata.setIssueManagement(convertIssueManagement(model.getIssueManagement()));
 351         metadata.setLicenses(convertLicenses(model.getLicenses()));
 352         metadata.setMailingLists(convertMailingLists(model.getMailingLists()));
 353         metadata.setDependencies(convertDependencies(model.getDependencies()));
 354         metadata.setName(model.getName());
 355         metadata.setOrganization(convertOrganization(model.getOrganization()));
 356         metadata.setScm(convertScm(model.getScm()));
 357         metadata.setUrl(model.getUrl());
 358         metadata.setProperties( new HashMap<String, String>( (Map) model.getProperties( ) ) );
 359
 360         MavenProjectFacet facet = new MavenProjectFacet();
 361         facet.setGroupId(model.getGroupId() != null ? model.getGroupId() : model.getParent().getGroupId());
 362         facet.setArtifactId(model.getArtifactId());
 363         facet.setPackaging(model.getPackaging());
 364         if (model.getParent() != null) {
 365             MavenProjectParent parent = new MavenProjectParent();
 366             parent.setGroupId(model.getParent().getGroupId());
 367             parent.setArtifactId(model.getParent().getArtifactId());
 368             parent.setVersion(model.getParent().getVersion());
 369             facet.setParent(parent);
 370         }
 371         metadata.addFacet(facet);
 372
 373         return metadata;
 374
 375
 376     }
 377
 378     public void setWagonFactory(WagonFactory wagonFactory) {
 379         this.wagonFactory = wagonFactory;
 380     }
 381
 382     private List<org.apache.archiva.metadata.model.Dependency> convertDependencies(List<Dependency> dependencies) {
 383         List<org.apache.archiva.metadata.model.Dependency> l = new ArrayList<>();
 384         for (Dependency dependency : dependencies) {
 385             org.apache.archiva.metadata.model.Dependency newDependency =
 386                     new org.apache.archiva.metadata.model.Dependency();
 387             newDependency.setArtifactId(dependency.getArtifactId());
 388             newDependency.setClassifier(dependency.getClassifier());
 389             newDependency.setNamespace(dependency.getGroupId());
 390             newDependency.setOptional(dependency.isOptional());
 391             newDependency.setScope(dependency.getScope());
 392             newDependency.setSystemPath(dependency.getSystemPath());
 393             newDependency.setType(dependency.getType());
 394             newDependency.setVersion(dependency.getVersion());
 395             l.add(newDependency);
 396         }
 397         return l;
 398     }
 399
 400     private org.apache.archiva.metadata.model.Scm convertScm(Scm scm) {
 401         org.apache.archiva.metadata.model.Scm newScm = null;
 402         if (scm != null) {
 403             newScm = new org.apache.archiva.metadata.model.Scm();
 404             newScm.setConnection(scm.getConnection());
 405             newScm.setDeveloperConnection(scm.getDeveloperConnection());
 406             newScm.setUrl(scm.getUrl());
 407         }
 408         return newScm;
 409     }
 410
 411     private org.apache.archiva.metadata.model.Organization convertOrganization(Organization organization) {
 412         org.apache.archiva.metadata.model.Organization org = null;
 413         if (organization != null) {
 414             org = new org.apache.archiva.metadata.model.Organization();
 415             org.setName(organization.getName());
 416             org.setUrl(organization.getUrl());
 417         }
 418         return org;
 419     }
 420
 421     private List<org.apache.archiva.metadata.model.License> convertLicenses(List<License> licenses) {
 422         List<org.apache.archiva.metadata.model.License> l = new ArrayList<>();
 423         for (License license : licenses) {
 424             org.apache.archiva.metadata.model.License newLicense = new org.apache.archiva.metadata.model.License();
 425             newLicense.setName(license.getName());
 426             newLicense.setUrl(license.getUrl());
 427             l.add(newLicense);
 428         }
 429         return l;
 430     }
 431
 432     private List<org.apache.archiva.metadata.model.MailingList> convertMailingLists(List<MailingList> mailingLists) {
 433         List<org.apache.archiva.metadata.model.MailingList> l = new ArrayList<>();
 434         for (MailingList mailingList : mailingLists) {
 435             org.apache.archiva.metadata.model.MailingList newMailingList =
 436                     new org.apache.archiva.metadata.model.MailingList();
 437             newMailingList.setName(mailingList.getName());
 438             newMailingList.setMainArchiveUrl(mailingList.getArchive());
 439             newMailingList.setPostAddress(mailingList.getPost());
 440             newMailingList.setSubscribeAddress(mailingList.getSubscribe());
 441             newMailingList.setUnsubscribeAddress(mailingList.getUnsubscribe());
 442             newMailingList.setOtherArchives(mailingList.getOtherArchives());
 443             l.add(newMailingList);
 444         }
 445         return l;
 446     }
 447
 448     private org.apache.archiva.metadata.model.IssueManagement convertIssueManagement(IssueManagement issueManagement) {
 449         org.apache.archiva.metadata.model.IssueManagement im = null;
 450         if (issueManagement != null) {
 451             im = new org.apache.archiva.metadata.model.IssueManagement();
 452             im.setSystem(issueManagement.getSystem());
 453             im.setUrl(issueManagement.getUrl());
 454         }
 455         return im;
 456     }
 457
 458     private org.apache.archiva.metadata.model.CiManagement convertCiManagement(CiManagement ciManagement) {
 459         org.apache.archiva.metadata.model.CiManagement ci = null;
 460         if (ciManagement != null) {
 461             ci = new org.apache.archiva.metadata.model.CiManagement();
 462             ci.setSystem(ciManagement.getSystem());
 463             ci.setUrl(ciManagement.getUrl());
 464         }
 465         return ci;
 466     }
 467
 468     @Override
 469     public Collection<String> listRootNamespaces(String repoId, Filter<String> filter)
 470             throws RepositoryStorageRuntimeException {
 471         StorageAsset dir = getRepositoryBasedir(repoId);
 472
 473         return getSortedFiles(dir, filter);
 474     }
 475
 476     private static Collection<String> getSortedFiles(StorageAsset dir, Filter<String> filter) {
 477
 478         final Predicate<StorageAsset> dFilter = new DirectoryFilter(filter);
 479         return dir.list().stream().filter(f -> f.isContainer())
 480                 .filter(dFilter)
 481                 .map(path -> path.getName().toString())
 482                 .sorted().collect(Collectors.toList());
 483
 484     }
 485
 486     private StorageAsset getRepositoryBasedir(String repoId)
 487             throws RepositoryStorageRuntimeException {
 488         ManagedRepository repository = repositoryRegistry.getManagedRepository(repoId);
 489
 490         return repository.getRoot();
 491     }
 492
 493     @Override
 494     public Collection<String> listNamespaces(String repoId, String namespace, Filter<String> filter)
 495             throws RepositoryStorageRuntimeException {
 496         StorageAsset dir = pathTranslator.toFile(getRepositoryBasedir(repoId), namespace);
 497         if (!(dir.exists()) && !dir.isContainer()) {
 498             return Collections.emptyList();
 499         }
 500         // scan all the directories which are potential namespaces. Any directories known to be projects are excluded
 501         Predicate<StorageAsset> dFilter = new DirectoryFilter(filter);
 502         return dir.list().stream().filter(dFilter).filter(path -> !isProject(path, filter)).map(path -> path.getName().toString())
 503                 .sorted().collect(Collectors.toList());
 504     }
 505
 506     @Override
 507     public Collection<String> listProjects(String repoId, String namespace, Filter<String> filter)
 508             throws RepositoryStorageRuntimeException {
 509         StorageAsset dir = pathTranslator.toFile(getRepositoryBasedir(repoId), namespace);
 510         if (!(dir.exists() && dir.isContainer())) {
 511             return Collections.emptyList();
 512         }
 513         // scan all directories in the namespace, and only include those that are known to be projects
 514         final Predicate<StorageAsset> dFilter = new DirectoryFilter(filter);
 515         return dir.list().stream().filter(dFilter).filter(path -> isProject(path, filter)).map(path -> path.getName().toString())
 516                 .sorted().collect(Collectors.toList());
 517
 518     }
 519
 520     @Override
 521     public Collection<String> listProjectVersions(String repoId, String namespace, String projectId,
 522                                                   Filter<String> filter)
 523             throws RepositoryStorageRuntimeException {
 524         StorageAsset dir = pathTranslator.toFile(getRepositoryBasedir(repoId), namespace, projectId);
 525         if (!(dir.exists() && dir.isContainer())) {
 526             return Collections.emptyList();
 527         }
 528
 529         // all directories in a project directory can be considered a version
 530         return getSortedFiles(dir, filter);
 531     }
 532
 533     @Override
 534     public Collection<ArtifactMetadata> readArtifactsMetadata(ReadMetadataRequest readMetadataRequest)
 535             throws RepositoryStorageRuntimeException {
 536         StorageAsset dir = pathTranslator.toFile(getRepositoryBasedir(readMetadataRequest.getRepositoryId()),
 537                 readMetadataRequest.getNamespace(), readMetadataRequest.getProjectId(),
 538                 readMetadataRequest.getProjectVersion());
 539         if (!(dir.exists() && dir.isContainer())) {
 540             return Collections.emptyList();
 541         }
 542
 543         // all files that are not metadata and not a checksum / signature are considered artifacts
 544         final Predicate<StorageAsset> dFilter = new ArtifactDirectoryFilter(readMetadataRequest.getFilter());
 545         // Returns a map TRUE -> (success values), FALSE -> (Exceptions)
 546         Map<Boolean, List<Try<ArtifactMetadata>>> result = dir.list().stream().filter(dFilter).map(path -> {
 547                     try {
 548                         return Try.success(getArtifactFromFile(readMetadataRequest.getRepositoryId(), readMetadataRequest.getNamespace(),
 549                                 readMetadataRequest.getProjectId(), readMetadataRequest.getProjectVersion(),
 550                                 path));
 551                     } catch (Exception e) {
 552                         log.debug("Could not create metadata for {}:  {}", path, e.getMessage(), e);
 553                         return Try.<ArtifactMetadata>failure(e);
 554                     }
 555                 }
 556         ).collect(Collectors.groupingBy(Try::isSuccess));
 557         if (result.containsKey(Boolean.FALSE) && result.get(Boolean.FALSE).size() > 0 && (!result.containsKey(Boolean.TRUE) || result.get(Boolean.TRUE).size() == 0)) {
 558             log.error("Could not get artifact metadata. Directory: {}. Number of errors {}.", dir, result.get(Boolean.FALSE).size());
 559             Try<ArtifactMetadata> failure = result.get(Boolean.FALSE).get(0);
 560             log.error("Sample exception {}", failure.getError().getMessage(), failure.getError());
 561             throw new RepositoryStorageRuntimeException(readMetadataRequest.getRepositoryId(), "Could not retrieve metadata of the files");
 562         } else {
 563             if (!result.containsKey(Boolean.TRUE) || result.get(Boolean.TRUE) == null) {
 564                 return Collections.emptyList();
 565             }
 566             return result.get(Boolean.TRUE).stream().map(tr -> tr.get()).collect(Collectors.toList());
 567         }
 568
 569     }
 570
 571     @Override
 572     public ArtifactMetadata readArtifactMetadataFromPath(String repoId, String path)
 573             throws RepositoryStorageRuntimeException {
 574         ArtifactMetadata metadata = pathTranslator.getArtifactForPath(repoId, path);
 575
 576         try {
 577             populateArtifactMetadataFromFile(metadata, getRepositoryBasedir(repoId).resolve(path));
 578         } catch (IOException e) {
 579             throw new RepositoryStorageRuntimeException(repoId, "Error during metadata retrieval of " + path + " :" + e.getMessage(), e);
 580         }
 581
 582         return metadata;
 583     }
 584
 585     private ArtifactMetadata getArtifactFromFile(String repoId, String namespace, String projectId,
 586                                                  String projectVersion, StorageAsset file) throws IOException {
 587         ArtifactMetadata metadata =
 588                 pathTranslator.getArtifactFromId(repoId, namespace, projectId, projectVersion, file.getName());
 589
 590         populateArtifactMetadataFromFile(metadata, file);
 591
 592         return metadata;
 593     }
 594
 595     @Override
 596     public ItemSelector applyServerSideRelocation(ManagedRepository managedRepository, ItemSelector artifactSelector)
 597         throws ProxyDownloadException {
 598         if ("pom".equals(artifactSelector.getType())) {
 599             return artifactSelector;
 600         }
 601
 602         // Build the artifact POM reference
 603         BaseRepositoryContentLayout layout;
 604         try
 605         {
 606             layout = managedRepository.getContent( ).getLayout( BaseRepositoryContentLayout.class );
 607         }
 608         catch ( LayoutException e )
 609         {
 610             throw new ProxyDownloadException( "Could not set layout " + e.getMessage( ), new HashMap<>(  ) );
 611         }
 612
 613         RepositoryType repositoryType = managedRepository.getType();
 614         if (!proxyRegistry.hasHandler(repositoryType)) {
 615             throw new ProxyDownloadException("No proxy handler found for repository type " + repositoryType, new HashMap<>());
 616         }
 617
 618
 619
 620         ItemSelector selector = ArchivaItemSelector.builder( )
 621             .withNamespace( artifactSelector.getNamespace( ) )
 622             .withProjectId( artifactSelector.getArtifactId( ) )
 623             .withArtifactId( artifactSelector.getArtifactId( ) )
 624             .withVersion( artifactSelector.getVersion( ) )
 625             .withArtifactVersion( artifactSelector.getVersion( ) )
 626             .withType( "pom" ).build( );
 627
 628         Artifact pom = layout.getArtifact( selector );
 629
 630         RepositoryProxyHandler proxyHandler = proxyRegistry.getHandler(repositoryType).get(0);
 631
 632         // Get the artifact POM from proxied repositories if needed
 633         proxyHandler.fetchFromProxies(managedRepository, pom);
 634
 635         // Open and read the POM from the managed repo
 636
 637         if (!pom.exists()) {
 638             return artifactSelector;
 639         }
 640
 641         try {
 642             // MavenXpp3Reader leaves the file open, so we need to close it ourselves.
 643
 644             Model model;
 645             try (Reader reader = Channels.newReader(pom.getAsset().getReadChannel(), Charset.defaultCharset().name())) {
 646                 model = MAVEN_XPP_3_READER.read(reader);
 647             }
 648
 649             DistributionManagement dist = model.getDistributionManagement();
 650             if (dist != null) {
 651                 Relocation relocation = dist.getRelocation();
 652                 if (relocation != null) {
 653                     ArchivaItemSelector.Builder relocatedBuilder = ArchivaItemSelector.builder( );
 654                     // artifact is relocated : update the repositoryPath
 655                     if (relocation.getGroupId() != null) {
 656                         relocatedBuilder.withNamespace( relocation.getGroupId( ) );
 657                     } else {
 658                         relocatedBuilder.withNamespace( artifactSelector.getNamespace( ) );
 659                     }
 660                     if (relocation.getArtifactId() != null) {
 661                         relocatedBuilder.withArtifactId( relocation.getArtifactId( ) );
 662                     } else {
 663                         relocatedBuilder.withArtifactId( artifactSelector.getArtifactId( ) );
 664                     }
 665                     if (relocation.getVersion() != null)
 666                     {
 667                         relocatedBuilder.withVersion( relocation.getVersion( ) );
 668                     } else {
 669                         relocatedBuilder.withVersion( artifactSelector.getVersion( ) );
 670                     }
 671                     return relocatedBuilder.withArtifactVersion( artifactSelector.getArtifactVersion( ) )
 672                         .withClassifier( artifactSelector.getClassifier( ) )
 673                         .withType( artifactSelector.getType( ) )
 674                         .withProjectId( artifactSelector.getProjectId( ) )
 675                         .withExtension( artifactSelector.getExtension( ) )
 676                         .build( );
 677                 }
 678             }
 679         } catch (IOException e) {
 680             // Unable to read POM : ignore.
 681         } catch (XmlPullParserException e) {
 682             // Invalid POM : ignore
 683         }
 684         return artifactSelector;
 685     }
 686
 687
 688     @Override
 689     public String getFilePath(String requestPath, org.apache.archiva.repository.ManagedRepository managedRepository) {
 690         // managedRepository can be null
 691         // extract artifact reference from url
 692         // groupId:artifactId:version:packaging:classifier
 693         //org/apache/archiva/archiva-checksum/1.4-M4-SNAPSHOT/archiva-checksum-1.4-M4-SNAPSHOT.jar
 694         String logicalResource = null;
 695         String requestPathInfo = StringUtils.defaultString(requestPath);
 696
 697         //remove prefix ie /repository/blah becomes /blah
 698         requestPathInfo = removePrefix(requestPathInfo);
 699
 700         // Remove prefixing slash as the repository id doesn't contain it;
 701         if (requestPathInfo.startsWith("/")) {
 702             requestPathInfo = requestPathInfo.substring(1);
 703         }
 704
 705         int slash = requestPathInfo.indexOf('/');
 706         if (slash > 0) {
 707             logicalResource = requestPathInfo.substring(slash);
 708
 709             if (logicalResource.endsWith("/..")) {
 710                 logicalResource += "/";
 711             }
 712
 713             if (logicalResource != null && logicalResource.startsWith("//")) {
 714                 logicalResource = logicalResource.substring(1);
 715             }
 716
 717             if (logicalResource == null) {
 718                 logicalResource = "/";
 719             }
 720         } else {
 721             logicalResource = "/";
 722         }
 723         return logicalResource;
 724
 725     }
 726
 727     @Override
 728     public String getFilePathWithVersion(final String requestPath, ManagedRepositoryContent managedRepositoryContent)
 729             throws RelocationException
 730     {
 731
 732         if (StringUtils.endsWith(requestPath, METADATA_FILENAME)) {
 733             return getFilePath(requestPath, managedRepositoryContent.getRepository());
 734         }
 735
 736         String filePath = getFilePath(requestPath, managedRepositoryContent.getRepository());
 737         Artifact artifact;
 738         try
 739         {
 740             ContentItem item = managedRepositoryContent.toItem( filePath );
 741             artifact = managedRepositoryContent.getLayout( BaseRepositoryContentLayout.class ).adaptItem( Artifact.class, item );
 742         }
 743         catch ( LayoutException e )
 744         {
 745             return filePath;
 746         }
 747
 748         if (VersionUtil.isGenericSnapshot(artifact.getArtifactVersion())) {
 749             // read maven metadata to get last timestamp
 750             StorageAsset metadataDir = managedRepositoryContent.getRepository().getAsset(filePath).getParent();
 751             if (!metadataDir.exists()) {
 752                 return filePath;
 753             }
 754             StorageAsset metadataFile = metadataDir.resolve(METADATA_FILENAME);
 755             if (!metadataFile.exists()) {
 756                 return filePath;
 757             }
 758             ArchivaRepositoryMetadata archivaRepositoryMetadata = null;
 759             try
 760             {
 761                 archivaRepositoryMetadata = metadataReader.read(metadataFile);
 762             }
 763             catch ( RepositoryMetadataException e )
 764             {
 765                 log.error( "Could not read metadata {}", e.getMessage( ), e );
 766                 return filePath;
 767             }
 768             int buildNumber = archivaRepositoryMetadata.getSnapshotVersion().getBuildNumber();
 769             String timestamp = archivaRepositoryMetadata.getSnapshotVersion().getTimestamp();
 770
 771             // MRM-1846
 772             if (buildNumber < 1 && timestamp == null) {
 773                 return filePath;
 774             }
 775
 776             // org/apache/archiva/archiva-checksum/1.4-M4-SNAPSHOT/archiva-checksum-1.4-M4-SNAPSHOT.jar
 777             // ->  archiva-checksum-1.4-M4-20130425.081822-1.jar
 778
 779             filePath = StringUtils.replace(filePath, //
 780                     artifact.getId() //
 781                             + "-" + artifact.getVersion().getId(), //
 782                     artifact.getId() //
 783                             + "-" + StringUtils.remove(artifact.getArtifactVersion(),
 784                             "-" + VersionUtil.SNAPSHOT) //
 785                             + "-" + timestamp //
 786                             + "-" + buildNumber);
 787
 788             throw new RelocationException("/repository/" + managedRepositoryContent.getRepository().getId() +
 789                     (StringUtils.startsWith(filePath, "/") ? "" : "/") + filePath,
 790                     RelocationException.RelocationType.TEMPORARY);
 791
 792         }
 793
 794         return filePath;
 795     }
 796
 797     //-----------------------------
 798     // internal
 799     //-----------------------------
 800
 801     /**
 802      * FIXME remove
 803      *
 804      * @param href
 805      * @return
 806      */
 807     private static String removePrefix(final String href) {
 808         String[] parts = StringUtils.split(href, '/');
 809         parts = (String[]) ArrayUtils.subarray(parts, 1, parts.length);
 810         if (parts == null || parts.length == 0) {
 811             return "/";
 812         }
 813
 814         String joinedString = StringUtils.join(parts, '/');
 815         if (href.endsWith("/")) {
 816             joinedString = joinedString + "/";
 817         }
 818
 819         return joinedString;
 820     }
 821
 822     private static void populateArtifactMetadataFromFile(ArtifactMetadata metadata, StorageAsset file) throws IOException {
 823         metadata.setWhenGathered(ZonedDateTime.now(ZoneId.of("GMT")));
 824         metadata.setFileLastModified(file.getModificationTime().toEpochMilli());
 825         ChecksummedFile checksummedFile = new ChecksummedFile(file.getFilePath());
 826         try {
 827             metadata.setMd5(checksummedFile.calculateChecksum(ChecksumAlgorithm.MD5));
 828         } catch (IOException e) {
 829             log.error("Unable to checksum file {}: {},MD5", file, e.getMessage());
 830         }
 831         try {
 832             metadata.setSha1(checksummedFile.calculateChecksum(ChecksumAlgorithm.SHA1));
 833         } catch (IOException e) {
 834             log.error("Unable to checksum file {}: {},SHA1", file, e.getMessage());
 835         }
 836         metadata.setSize(file.getSize());
 837     }
 838
 839     private boolean isProject(StorageAsset dir, Filter<String> filter) {
 840         // scan directories for a valid project version subdirectory, meaning this must be a project directory
 841         final Predicate<StorageAsset> dFilter = new DirectoryFilter(filter);
 842         boolean projFound = dir.list().stream().filter(dFilter)
 843                 .anyMatch(path -> isProjectVersion(path));
 844         if (projFound) {
 845             return true;
 846         }
 847
 848         // if a metadata file is present, check if this is the "artifactId" directory, marking it as a project
 849         ArchivaRepositoryMetadata metadata = readMetadata(dir);
 850         if (metadata != null && dir.getName().toString().equals(metadata.getArtifactId())) {
 851             return true;
 852         }
 853
 854         return false;
 855     }
 856
 857     private boolean isProjectVersion(StorageAsset dir) {
 858         final String artifactId = dir.getParent().getName();
 859         final String projectVersion = dir.getName();
 860
 861         // check if there is a POM artifact file to ensure it is a version directory
 862
 863         Predicate<StorageAsset> filter;
 864         if (VersionUtil.isSnapshot(projectVersion)) {
 865             filter = new PomFilenameFilter(artifactId, projectVersion);
 866         } else {
 867             final String pomFile = artifactId + "-" + projectVersion + ".pom";
 868             filter = new PomFileFilter(pomFile);
 869         }
 870         if (dir.list().stream().filter(f -> !f.isContainer()).anyMatch(filter)) {
 871             return true;
 872         }
 873         // if a metadata file is present, check if this is the "version" directory, marking it as a project version
 874         ArchivaRepositoryMetadata metadata = readMetadata(dir);
 875         if (metadata != null && projectVersion.equals(metadata.getVersion())) {
 876             return true;
 877         }
 878
 879         return false;
 880     }
 881
 882     private ArchivaRepositoryMetadata readMetadata(StorageAsset directory) {
 883         ArchivaRepositoryMetadata metadata = null;
 884         StorageAsset metadataFile = directory.resolve(METADATA_FILENAME);
 885         if (metadataFile.exists()) {
 886             try {
 887                 metadata = metadataReader.read(metadataFile);
 888             } catch ( RepositoryMetadataException e )
 889             {
 890                 // Ignore missing or invalid metadata
 891             }
 892         }
 893         return metadata;
 894     }
 895
 896     private static class DirectoryFilter
 897             implements Predicate<StorageAsset> {
 898         private final Filter<String> filter;
 899
 900         public DirectoryFilter(Filter<String> filter) {
 901             this.filter = filter;
 902         }
 903
 904         @Override
 905         public boolean test(StorageAsset dir) {
 906             final String name = dir.getName();
 907             if (!filter.accept(name)) {
 908                 return false;
 909             } else if (name.startsWith(".")) {
 910                 return false;
 911             } else if (!dir.isContainer()) {
 912                 return false;
 913             }
 914             return true;
 915         }
 916     }
 917
 918     private static class ArtifactDirectoryFilter
 919             implements Predicate<StorageAsset> {
 920         private final Filter<String> filter;
 921
 922         private ArtifactDirectoryFilter(Filter<String> filter) {
 923             this.filter = filter;
 924         }
 925
 926         @Override
 927         public boolean test(StorageAsset file) {
 928             final Set<String> checksumExts = ChecksumAlgorithm.getAllExtensions();
 929             final String path = file.getPath();
 930             final String name = file.getName();
 931             final String extension = StringUtils.substringAfterLast(name, ".").toLowerCase();
 932             // TODO compare to logic in maven-repository-layer
 933             if (file.isContainer()) {
 934                 return false;
 935             } else if (!filter.accept(name)) {
 936                 return false;
 937             } else if (name.startsWith(".") || path.contains("/.") ) {
 938                 return false;
 939             } else if (checksumExts.contains(extension)) {
 940                 return false;
 941             } else if (Arrays.binarySearch(IGNORED_FILES, name) >= 0) {
 942                 return false;
 943             }
 944             // some files from remote repositories can have name like maven-metadata-archiva-vm-all-public.xml
 945             else if (StringUtils.startsWith(name, METADATA_FILENAME_START) && StringUtils.endsWith(name, ".xml")) {
 946                 return false;
 947             }
 948
 949             return true;
 950
 951         }
 952     }
 953
 954
 955     private static final class PomFilenameFilter
 956             implements Predicate<StorageAsset> {
 957
 958         private final String artifactId, projectVersion;
 959
 960         private PomFilenameFilter(String artifactId, String projectVersion) {
 961             this.artifactId = artifactId;
 962             this.projectVersion = projectVersion;
 963         }
 964
 965         @Override
 966         public boolean test(StorageAsset dir) {
 967             final String name = dir.getName();
 968             if (name.startsWith(artifactId + "-") && name.endsWith(".pom")) {
 969                 String v = name.substring(artifactId.length() + 1, name.length() - 4);
 970                 v = VersionUtil.getBaseVersion(v);
 971                 if (v.equals(projectVersion)) {
 972                     return true;
 973                 }
 974             }
 975             return false;
 976         }
 977
 978     }
 979
 980     private static class PomFileFilter
 981             implements Predicate<StorageAsset> {
 982         private final String pomFile;
 983
 984         private PomFileFilter(String pomFile) {
 985             this.pomFile = pomFile;
 986         }
 987
 988         @Override
 989         public boolean test(StorageAsset dir) {
 990             return pomFile.equals(dir.getName());
 991         }
 992     }
 993
 994 }