source.dussan.org Git - archiva.git/blob

   1 package org.apache.archiva.repository.scanner;
   2
   3 /*
   4  * Licensed to the Apache Software Foundation (ASF) under one
   5  * or more contributor license agreements.  See the NOTICE file
   6  * distributed with this work for additional information
   7  * regarding copyright ownership.  The ASF licenses this file
   8  * to you under the Apache License, Version 2.0 (the
   9  * "License"); you may not use this file except in compliance
  10  * with the License.  You may obtain a copy of the License at
  11  *
  12  *  http://www.apache.org/licenses/LICENSE-2.0
  13  *
  14  * Unless required by applicable law or agreed to in writing,
  15  * software distributed under the License is distributed on an
  16  * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
  17  * KIND, either express or implied.  See the License for the
  18  * specific language governing permissions and limitations
  19  * under the License.
  20  */
  21
  22 import org.apache.archiva.common.utils.BaseFile;
  23 import org.apache.archiva.common.utils.PathUtil;
  24 import org.apache.archiva.consumers.InvalidRepositoryContentConsumer;
  25 import org.apache.archiva.consumers.KnownRepositoryContentConsumer;
  26 import org.apache.archiva.consumers.functors.ConsumerWantsFilePredicate;
  27 import org.apache.archiva.repository.ManagedRepository;
  28 import org.apache.archiva.repository.scanner.functors.ConsumerProcessFileClosure;
  29 import org.apache.archiva.repository.scanner.functors.TriggerBeginScanClosure;
  30 import org.apache.archiva.repository.scanner.functors.TriggerScanCompletedClosure;
  31 import org.apache.commons.collections.Closure;
  32 import org.apache.commons.collections.CollectionUtils;
  33 import org.apache.commons.collections.functors.IfClosure;
  34 import org.apache.commons.lang.SystemUtils;
  35 import org.slf4j.Logger;
  36 import org.slf4j.LoggerFactory;
  37
  38 import java.io.IOException;
  39 import java.nio.file.FileSystem;
  40 import java.nio.file.FileSystems;
  41 import java.nio.file.FileVisitResult;
  42 import java.nio.file.FileVisitor;
  43 import java.nio.file.Files;
  44 import java.nio.file.Path;
  45 import java.nio.file.PathMatcher;
  46 import java.nio.file.attribute.BasicFileAttributes;
  47 import java.util.ArrayList;
  48 import java.util.Date;
  49 import java.util.HashMap;
  50 import java.util.List;
  51 import java.util.Map;
  52 import java.util.stream.Collectors;
  53
  54 /**
  55  * RepositoryScannerInstance
  56  */
  57 public class RepositoryScannerInstance
  58     implements FileVisitor<Path>
  59 {
  60     private Logger log = LoggerFactory.getLogger( RepositoryScannerInstance.class );
  61
  62     /**
  63      * Consumers that process known content.
  64      */
  65     private List<KnownRepositoryContentConsumer> knownConsumers;
  66
  67     /**
  68      * Consumers that process unknown/invalid content.
  69      */
  70     private List<InvalidRepositoryContentConsumer> invalidConsumers;
  71
  72     private ManagedRepository repository;
  73
  74     private RepositoryScanStatistics stats;
  75
  76     private long changesSince = 0;
  77
  78     private ConsumerProcessFileClosure consumerProcessFile;
  79
  80     private ConsumerWantsFilePredicate consumerWantsFile;
  81
  82     private Map<String, Long> consumerTimings;
  83
  84     private Map<String, Long> consumerCounts;
  85
  86
  87     private List<String> fileNameIncludePattern = new ArrayList<>();
  88     private List<String> fileNameExcludePattern = new ArrayList<>();
  89
  90     private List<PathMatcher> includeMatcher = new ArrayList<>();
  91     private List<PathMatcher> excludeMatcher = new ArrayList<>();
  92
  93     private boolean isRunning = false;
  94
  95     Path basePath = null;
  96
  97     public RepositoryScannerInstance( ManagedRepository repository,
  98                                       List<KnownRepositoryContentConsumer> knownConsumerList,
  99                                       List<InvalidRepositoryContentConsumer> invalidConsumerList )
 100     {
 101         this.repository = repository;
 102         this.knownConsumers = knownConsumerList;
 103         this.invalidConsumers = invalidConsumerList;
 104
 105         addFileNameIncludePattern("**/*");
 106
 107         consumerTimings = new HashMap<>();
 108         consumerCounts = new HashMap<>();
 109
 110         this.consumerProcessFile = new ConsumerProcessFileClosure();
 111         consumerProcessFile.setExecuteOnEntireRepo( true );
 112         consumerProcessFile.setConsumerTimings( consumerTimings );
 113         consumerProcessFile.setConsumerCounts( consumerCounts );
 114
 115         this.consumerWantsFile = new ConsumerWantsFilePredicate( repository );
 116
 117         stats = new RepositoryScanStatistics();
 118         stats.setRepositoryId( repository.getId() );
 119
 120         Closure triggerBeginScan =
 121             new TriggerBeginScanClosure( repository, new Date( System.currentTimeMillis() ), true );
 122
 123         CollectionUtils.forAllDo( knownConsumerList, triggerBeginScan );
 124         CollectionUtils.forAllDo( invalidConsumerList, triggerBeginScan );
 125
 126         if ( SystemUtils.IS_OS_WINDOWS )
 127         {
 128             consumerWantsFile.setCaseSensitive( false );
 129         }
 130     }
 131
 132     public RepositoryScannerInstance( ManagedRepository repository,
 133                                       List<KnownRepositoryContentConsumer> knownContentConsumers,
 134                                       List<InvalidRepositoryContentConsumer> invalidContentConsumers,
 135                                       long changesSince )
 136     {
 137         this( repository, knownContentConsumers, invalidContentConsumers );
 138
 139         consumerWantsFile.setChangesSince( changesSince );
 140
 141         this.changesSince = changesSince;
 142     }
 143
 144     public RepositoryScanStatistics getStatistics()
 145     {
 146         return stats;
 147     }
 148
 149     public Map<String, Long> getConsumerTimings()
 150     {
 151         return consumerTimings;
 152     }
 153
 154     public Map<String, Long> getConsumerCounts()
 155     {
 156         return consumerCounts;
 157     }
 158
 159     public ManagedRepository getRepository()
 160     {
 161         return repository;
 162     }
 163
 164     public RepositoryScanStatistics getStats()
 165     {
 166         return stats;
 167     }
 168
 169     public long getChangesSince()
 170     {
 171         return changesSince;
 172     }
 173
 174     public List<String> getFileNameIncludePattern() {
 175         return fileNameIncludePattern;
 176     }
 177
 178     public void setFileNameIncludePattern(List<String> fileNamePattern) {
 179         this.fileNameIncludePattern = fileNamePattern;
 180         FileSystem sys = FileSystems.getDefault();
 181         this.includeMatcher = fileNamePattern.stream().map(ts ->sys
 182                 .getPathMatcher("glob:" + ts)).collect(Collectors.toList());
 183     }
 184
 185     public void addFileNameIncludePattern(String fileNamePattern) {
 186         if (! this.fileNameIncludePattern.contains(fileNamePattern)) {
 187             this.fileNameIncludePattern.add(fileNamePattern);
 188             this.includeMatcher.add(FileSystems.getDefault().getPathMatcher("glob:" + fileNamePattern));
 189         }
 190     }
 191
 192     public List<String> getFileNameExcludePattern() {
 193         return fileNameExcludePattern;
 194     }
 195
 196     public void setFileNameExcludePattern(List<String> fileNamePattern) {
 197         this.fileNameExcludePattern = fileNamePattern;
 198         FileSystem sys = FileSystems.getDefault();
 199         this.excludeMatcher = fileNamePattern.stream().map(ts ->sys
 200                 .getPathMatcher("glob:" + ts)).collect(Collectors.toList());
 201     }
 202
 203     public void addFileNameExcludePattern(String fileNamePattern) {
 204         if (! this.fileNameExcludePattern.contains(fileNamePattern)) {
 205             this.fileNameExcludePattern.add(fileNamePattern);
 206             this.excludeMatcher.add(FileSystems.getDefault().getPathMatcher("glob:" + fileNamePattern));
 207         }
 208     }
 209
 210
 211     @Override
 212     public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException {
 213         if (!isRunning) {
 214             isRunning = true;
 215             this.basePath = dir;
 216             log.info( "Walk Started: [{}] {}", this.repository.getId(), this.repository.getLocation() );
 217             stats.triggerStart();
 218         }
 219         return FileVisitResult.CONTINUE;
 220     }
 221
 222     @Override
 223     public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
 224         if (excludeMatcher.stream().noneMatch(m -> m.matches(file)) && includeMatcher.stream().allMatch(m -> m.matches(file))) {
 225             log.debug( "Walk Step: {}, {}", file );
 226
 227             stats.increaseFileCount();
 228
 229             // consume files regardless - the predicate will check the timestamp
 230             Path repoPath = PathUtil.getPathFromUri( repository.getLocation() );
 231             BaseFile basefile = new BaseFile( repoPath.toString(), file.toFile() );
 232
 233             // Timestamp finished points to the last successful scan, not this current one.
 234             if ( Files.getLastModifiedTime(file).toMillis() >= changesSince )
 235             {
 236                 stats.increaseNewFileCount();
 237             }
 238
 239             consumerProcessFile.setBasefile( basefile );
 240             consumerWantsFile.setBasefile( basefile );
 241
 242             Closure processIfWanted = IfClosure.getInstance( consumerWantsFile, consumerProcessFile );
 243             CollectionUtils.forAllDo( this.knownConsumers, processIfWanted );
 244
 245             if ( consumerWantsFile.getWantedFileCount() <= 0 )
 246             {
 247                 // Nothing known processed this file.  It is invalid!
 248                 CollectionUtils.forAllDo( this.invalidConsumers, consumerProcessFile );
 249             }
 250
 251         }
 252         return FileVisitResult.CONTINUE;
 253     }
 254
 255     @Override
 256     public FileVisitResult visitFileFailed(Path file, IOException exc) throws IOException {
 257         log.error("Error occured at {}: {}", file, exc.getMessage(), exc);
 258         if (basePath!=null && Files.isSameFile(file, basePath)) {
 259             finishWalk();
 260         }
 261         return FileVisitResult.CONTINUE;
 262     }
 263
 264     @Override
 265     public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException {
 266         if (Files.isSameFile(dir, basePath)) {
 267             finishWalk();
 268         }
 269         return FileVisitResult.CONTINUE;
 270     }
 271
 272     private void finishWalk() {
 273         this.isRunning = false;
 274         TriggerScanCompletedClosure scanCompletedClosure = new TriggerScanCompletedClosure( repository, true );
 275         CollectionUtils.forAllDo( knownConsumers, scanCompletedClosure );
 276         CollectionUtils.forAllDo( invalidConsumers, scanCompletedClosure );
 277
 278         stats.setConsumerTimings( consumerTimings );
 279         stats.setConsumerCounts( consumerCounts );
 280
 281         log.info( "Walk Finished: [{}] {}", this.repository.getId(), this.repository.getLocation() );
 282         stats.triggerFinished();
 283         this.basePath = null;
 284     }
 285 }