import org.sonar.xoo.coverage.ItCoverageSensor;
import org.sonar.xoo.coverage.OverallCoverageSensor;
import org.sonar.xoo.coverage.UtCoverageSensor;
+import org.sonar.xoo.extensions.XooExcludeFileFilter;
import org.sonar.xoo.extensions.XooIssueFilter;
import org.sonar.xoo.extensions.XooPostJob;
import org.sonar.xoo.extensions.XooProjectBuilder;
XooIssueFilter.class,
XooIgnoreCommand.class,
SignificantCodeSensor.class,
- IssueWithCodeVariantsSensor.class);
+ IssueWithCodeVariantsSensor.class,
+ XooExcludeFileFilter.class);
if (context.getRuntime().getProduct() != SonarProduct.SONARLINT) {
context.addExtension(MeasureSensor.class);
--- /dev/null
+/*
+ * SonarQube
+ * Copyright (C) 2009-2023 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+package org.sonar.xoo.extensions;
+
+import org.sonar.api.batch.fs.InputFile;
+import org.sonar.api.batch.fs.InputFileFilter;
+
+public class XooExcludeFileFilter implements InputFileFilter {
+
+ @Override
+ public boolean accept(InputFile f) {
+ return !f.filename().endsWith("_exclude.xoo");
+ }
+}
throw new IllegalStateException(String.format("The classloader '%s' already exists in the list of previously created classloaders."
+ " Can not create it twice.", key));
}
- ClassRealm realm = AccessController.<PrivilegedAction<ClassRealm>>doPrivileged(() -> new ClassRealm(key, baseClassloader));
+ //TODO: to be checked, the other version of the code is not building
+ ClassRealm realm = AccessController.doPrivileged(new PrivilegedAction<ClassRealm>() {
+ @Override
+ public ClassRealm run() {
+ return new ClassRealm(key, baseClassloader);
+ }
+ });
realm.setStrategy(LoadingOrder.PARENT_FIRST.strategy);
newRealmsByKey.put(key, new NewRealm(realm));
return this;
import java.util.Map;
import java.util.Optional;
import java.util.Properties;
+import java.util.Set;
import javax.annotation.CheckForNull;
import javax.annotation.Nullable;
import javax.annotation.Priority;
}
public ScannerMediumTester registerPlugin(String pluginKey, Plugin instance) {
- return registerPlugin(pluginKey, instance, 1L);
+ pluginInstaller.add(pluginKey, instance);
+ return this;
}
- public ScannerMediumTester registerPlugin(String pluginKey, Plugin instance, long lastUpdatedAt) {
- pluginInstaller.add(pluginKey, instance, lastUpdatedAt);
+ public ScannerMediumTester registerOptionalPlugin(String pluginKey, Set<String> requiredForLanguages, Plugin instance) {
+ pluginInstaller.addOptional(pluginKey, requiredForLanguages, instance);
return this;
}
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Random;
+import java.util.Set;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang.SystemUtils;
import org.junit.rules.TemporaryFolder;
import org.slf4j.event.Level;
import org.sonar.api.CoreProperties;
+import org.sonar.api.Plugin;
import org.sonar.api.SonarEdition;
import org.sonar.api.batch.fs.InputFile;
+import org.sonar.api.batch.fs.InputFileFilter;
import org.sonar.api.batch.fs.internal.DefaultInputFile;
import org.sonar.api.testfixtures.log.LogTester;
import org.sonar.api.utils.MessageException;
public ScannerMediumTester tester = new ScannerMediumTester()
.setEdition(SonarEdition.COMMUNITY)
.registerPlugin("xoo", new XooPlugin())
+ .registerOptionalPlugin("optional-xoo", Set.of("xoo"), new OptionalXooPlugin())
.addDefaultQProfile("xoo", "Sonar Way")
.addDefaultQProfile("xoo2", "Sonar Way");
assertThatThrownBy(result::execute)
.isExactlyInstanceOf(IllegalStateException.class)
- .hasMessageEndingWith(format("Failed to index files"));
+ .hasMessageEndingWith(format("Failed to preprocess files"));
}
@Test
assertThatThrownBy(result::execute)
.isExactlyInstanceOf(IllegalStateException.class)
- .hasMessageEndingWith(format("Failed to index files"));
+ .hasMessageEndingWith(format("Failed to preprocess files"));
+ }
+
+ @Test
+ public void should_load_input_file_filters_for_required_and_optional_plugins() throws IOException {
+ File projectDir = new File("test-resources/mediumtest/xoo/sample-with-input-file-filters");
+ AnalysisResult result = tester
+ .newAnalysis(new File(projectDir, "sonar-project.properties"))
+ .execute();
+
+ assertThat(result.inputFiles()).hasSize(1);
+
+ assertThat(logTester.logs()).contains("'xources/hello/xoo_exclude2.xoo' excluded by org.sonar.scanner.mediumtest.fs" +
+ ".FileSystemMediumIT$OptionalXooPlugin$OptionalXooFileFilter");
+ assertThat(logTester.logs()).contains("'xources/hello/xoo_exclude.xoo' excluded by org.sonar.xoo.extensions.XooExcludeFileFilter");
+ assertThat(logTester.logs()).contains("'xources/hello/HelloJava.xoo' indexed with language 'xoo'");
+
+ assertThat(result.inputFile("xources/hello/xoo_exclude.xoo")).isNull();
+ assertThat(result.inputFile("xources/hello/xoo_exclude2.xoo")).isNull();
+ assertThat(result.inputFile("xources/hello/HelloJava.xoo")).isNotNull();
+ }
+
+ public static class OptionalXooPlugin implements Plugin {
+
+ @Override
+ public void define(Context context) {
+ context.addExtension(OptionalXooFileFilter.class);
+ }
+
+ public static class OptionalXooFileFilter implements InputFileFilter {
+
+ @Override
+ public boolean accept(InputFile f) {
+ return !f.filename().endsWith("_exclude2.xoo");
+ }
+ }
}
private static void assertAnalysedFiles(AnalysisResult result, String... files) {
import java.util.List;
import java.util.Map;
import java.util.Set;
+import org.sonar.scanner.mediumtest.LocalPlugin;
public interface PluginInstaller {
Map<String, ScannerPlugin> installPluginsForLanguages(Set<String> languageKeys);
/**
- * Used only by medium tests.
+ * Used only by medium tests. Installs required plugins (phase 1)
* @see org.sonar.scanner.mediumtest.ScannerMediumTester
*/
- List<Object[]> installLocals();
+ List<LocalPlugin> installLocals();
+
+ /**
+ * Used only by medium tests. Installs optional plugins (phase 2)
+ * @see org.sonar.scanner.mediumtest.ScannerMediumTester
+ */
+ List<LocalPlugin> installOptionalLocals(Set<String> languageKeys);
}
import org.sonar.api.utils.log.Profiler;
import org.sonar.core.platform.PluginInfo;
import org.sonar.core.plugin.PluginType;
+import org.sonar.scanner.mediumtest.LocalPlugin;
import org.sonarqube.ws.client.GetRequest;
import static java.lang.String.format;
* Returns empty on purpose. This method is used only by medium tests.
*/
@Override
- public List<Object[]> installLocals() {
+ public List<LocalPlugin> installLocals() {
+ return Collections.emptyList();
+ }
+
+ /**
+ * Returns empty on purpose. This method is used only by medium tests.
+ */
+ @Override
+ public List<LocalPlugin> installOptionalLocals(Set<String> languageKeys) {
return Collections.emptyList();
}
import org.sonar.core.platform.PluginJarExploder;
import org.sonar.core.platform.PluginRepository;
import org.sonar.core.plugin.PluginType;
+import org.sonar.scanner.mediumtest.LocalPlugin;
import static java.util.stream.Collectors.toMap;
import static org.sonar.api.utils.Preconditions.checkState;
pluginInstancesByKeys = new HashMap<>(loader.load(explodedPluginsByKey));
// this part is only used by medium tests
- for (Object[] localPlugin : installer.installLocals()) {
- String pluginKey = (String) localPlugin[0];
- PluginInfo pluginInfo = new PluginInfo(pluginKey);
- pluginsByKeys.put(pluginKey, new ScannerPlugin(pluginInfo.getKey(), (long) localPlugin[2], PluginType.BUNDLED, pluginInfo));
- pluginInstancesByKeys.put(pluginKey, (Plugin) localPlugin[1]);
+ for (LocalPlugin localPlugin : installer.installLocals()) {
+ ScannerPlugin scannerPlugin = localPlugin.toScannerPlugin();
+ String pluginKey = localPlugin.pluginKey();
+ pluginsByKeys.put(pluginKey, scannerPlugin);
+ pluginInstancesByKeys.put(pluginKey, localPlugin.pluginInstance());
}
keysByClassLoader = new HashMap<>();
.collect(toMap(Map.Entry::getKey, e -> pluginJarExploder.explode(e.getValue().getInfo())));
pluginInstancesByKeys.putAll(new HashMap<>(loader.load(explodedPluginsByKey)));
+ // this part is only used by medium tests
+ for (LocalPlugin localPlugin : installer.installOptionalLocals(languageKeys)) {
+ ScannerPlugin scannerPlugin = localPlugin.toScannerPlugin();
+ String pluginKey = localPlugin.pluginKey();
+ languagePluginsByKeys.put(pluginKey, scannerPlugin);
+ pluginsByKeys.put(pluginKey, scannerPlugin);
+ pluginInstancesByKeys.put(pluginKey, localPlugin.pluginInstance());
+ }
+
keysByClassLoader = new HashMap<>();
for (Map.Entry<String, Plugin> e : pluginInstancesByKeys.entrySet()) {
keysByClassLoader.put(e.getValue().getClass().getClassLoader(), e.getKey());
import org.sonar.scanner.scan.branch.BranchType;
import org.sonar.scanner.scan.branch.ProjectBranchesProvider;
import org.sonar.scanner.scan.filesystem.DefaultProjectFileSystem;
-import org.sonar.scanner.scan.filesystem.FileIndexer;
+import org.sonar.scanner.scan.filesystem.FilePreprocessor;
import org.sonar.scanner.scan.filesystem.InputComponentStore;
import org.sonar.scanner.scan.filesystem.LanguageDetection;
import org.sonar.scanner.scan.filesystem.MetadataGenerator;
+import org.sonar.scanner.scan.filesystem.ModuleRelativePathWarner;
import org.sonar.scanner.scan.filesystem.ProjectCoverageAndDuplicationExclusions;
import org.sonar.scanner.scan.filesystem.ProjectExclusionFilters;
-import org.sonar.scanner.scan.filesystem.ProjectFileIndexer;
+import org.sonar.scanner.scan.filesystem.ProjectFilePreprocessor;
import org.sonar.scanner.scan.filesystem.ScannerComponentIdGenerator;
import org.sonar.scanner.scan.filesystem.StatusDetection;
import org.sonar.scanner.scan.measure.DefaultMetricFinder;
LanguageDetection.class,
MetadataGenerator.class,
FileMetadata.class,
- FileIndexer.class,
- ProjectFileIndexer.class,
+ ModuleRelativePathWarner.class,
+ FilePreprocessor.class,
+ ProjectFilePreprocessor.class,
ProjectExclusionFilters.class,
// rules
getComponentByType(DeprecatedPropertiesWarningGenerator.class).execute();
- getComponentByType(ProjectFileIndexer.class).index();
+ getComponentByType(ProjectFilePreprocessor.class).execute();
new SpringProjectScanContainer(this).execute();
}
public class FakePluginInstaller implements PluginInstaller {
private final Map<String, ScannerPlugin> pluginsByKeys = new HashMap<>();
- private final List<Object[]> mediumTestPlugins = new ArrayList<>();
+ private final List<LocalPlugin> mediumTestPlugins = new ArrayList<>();
+ private final List<LocalPlugin> optionalMediumTestPlugins = new ArrayList<>();
public FakePluginInstaller add(String pluginKey, File jarFile, long lastUpdatedAt) {
pluginsByKeys.put(pluginKey, new ScannerPlugin(pluginKey, lastUpdatedAt, PluginType.BUNDLED, PluginInfo.create(jarFile)));
return this;
}
- public FakePluginInstaller add(String pluginKey, Plugin instance, long lastUpdatedAt) {
- mediumTestPlugins.add(new Object[] {pluginKey, instance, lastUpdatedAt});
+ public FakePluginInstaller add(String pluginKey, Plugin instance) {
+ mediumTestPlugins.add(new LocalPlugin(pluginKey, instance, Set.of()));
+ return this;
+ }
+
+ public FakePluginInstaller addOptional(String pluginKey, Set<String> requiredForLanguages, Plugin instance) {
+ optionalMediumTestPlugins.add(new LocalPlugin(pluginKey, instance, requiredForLanguages));
return this;
}
}
@Override
- public List<Object[]> installLocals() {
+ public List<LocalPlugin> installLocals() {
return mediumTestPlugins;
}
+
+ @Override
+ public List<LocalPlugin> installOptionalLocals(Set<String> languageKeys) {
+ return optionalMediumTestPlugins.stream()
+ .filter(plugin -> languageKeys.stream().anyMatch(lang -> plugin.requiredForLanguages().contains(lang)))
+ .toList();
+ }
}
--- /dev/null
+/*
+ * SonarQube
+ * Copyright (C) 2009-2023 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+package org.sonar.scanner.mediumtest;
+
+import java.util.Set;
+import org.sonar.api.Plugin;
+import org.sonar.core.platform.PluginInfo;
+import org.sonar.core.plugin.PluginType;
+import org.sonar.scanner.bootstrap.ScannerPlugin;
+
+public record LocalPlugin(String pluginKey, Plugin pluginInstance, Set<String> requiredForLanguages) {
+
+ public ScannerPlugin toScannerPlugin() {
+ return new ScannerPlugin(pluginKey, 1L, PluginType.BUNDLED, new PluginInfo(pluginKey));
+ }
+}
import org.sonar.scanner.report.ContextPropertiesPublisher;
import org.sonar.scanner.report.ReportPublisher;
import org.sonar.scanner.rule.QProfileVerifier;
-import org.sonar.scanner.scan.filesystem.InputComponentStore;
+import org.sonar.scanner.scan.filesystem.FileIndexer;
+import org.sonar.scanner.scan.filesystem.InputFileFilterRepository;
+import org.sonar.scanner.scan.filesystem.LanguageDetection;
+import org.sonar.scanner.scan.filesystem.ProjectFileIndexer;
import org.sonar.scanner.scm.ScmPublisher;
import org.sonar.scanner.sensor.ProjectSensorExtensionDictionary;
import org.sonar.scanner.sensor.ProjectSensorsExecutor;
@Override
protected void doBeforeStart() {
- Set<String> languages = getParentComponentByType(InputComponentStore.class).languages();
+ Set<String> languages = getParentComponentByType(LanguageDetection.class).getDetectedLanguages();
installPluginsForLanguages(languages);
addScannerComponents();
}
ProjectSensorExtensionDictionary.class,
ProjectSensorsExecutor.class,
- AnalysisObservers.class);
+ AnalysisObservers.class,
+
+ // file system
+ InputFileFilterRepository.class,
+ FileIndexer.class,
+ ProjectFileIndexer.class);
}
static ExtensionMatcher getScannerProjectExtensionsFilter() {
protected void doAfterStart() {
getParentComponentByType(ScannerMetrics.class).addPluginMetrics(getComponentsByType(Metrics.class));
getComponentByType(ProjectLock.class).tryLock();
+ getComponentByType(ProjectFileIndexer.class).index();
GlobalAnalysisMode analysisMode = getComponentByType(GlobalAnalysisMode.class);
InputModuleHierarchy tree = getComponentByType(InputModuleHierarchy.class);
ScanProperties properties = getComponentByType(ScanProperties.class);
--- /dev/null
+/*
+ * SonarQube
+ * Copyright (C) 2009-2023 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+package org.sonar.scanner.scan.filesystem;
+
+import java.io.IOException;
+import java.nio.file.AccessDeniedException;
+import java.nio.file.FileSystemLoopException;
+import java.nio.file.FileVisitResult;
+import java.nio.file.FileVisitor;
+import java.nio.file.Files;
+import java.nio.file.LinkOption;
+import java.nio.file.Path;
+import java.nio.file.attribute.BasicFileAttributes;
+import java.nio.file.attribute.DosFileAttributes;
+import org.apache.commons.lang.SystemUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.sonar.api.batch.fs.InputFile;
+import org.sonar.api.batch.fs.internal.DefaultInputModule;
+import org.sonar.scanner.fs.InputModuleHierarchy;
+
+public class DirectoryFileVisitor implements FileVisitor<Path> {
+
+ private static final Logger LOG = LoggerFactory.getLogger(DirectoryFileVisitor.class);
+
+ private final FileVisitAction fileVisitAction;
+ private final DefaultInputModule module;
+ private final ModuleExclusionFilters moduleExclusionFilters;
+
+ private final InputModuleHierarchy inputModuleHierarchy;
+ private final InputFile.Type type;
+
+ DirectoryFileVisitor(FileVisitAction fileVisitAction, DefaultInputModule module, ModuleExclusionFilters moduleExclusionFilters, InputModuleHierarchy inputModuleHierarchy, InputFile.Type type) {
+ this.fileVisitAction = fileVisitAction;
+ this.module = module;
+ this.moduleExclusionFilters = moduleExclusionFilters;
+ this.inputModuleHierarchy = inputModuleHierarchy;
+ this.type = type;
+ }
+
+ @Override
+ public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException {
+ return isHidden(dir) ? FileVisitResult.SKIP_SUBTREE : FileVisitResult.CONTINUE;
+ }
+
+ @Override
+ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
+ if (!Files.isHidden(file)) {
+ fileVisitAction.execute(file);
+ }
+ return FileVisitResult.CONTINUE;
+ }
+
+ /**
+ * <p>Overridden method to handle exceptions while visiting files in the analysis.</p>
+ *
+ * <p>
+ * <ul>
+ * <li>FileSystemLoopException - We show a warning that a symlink loop exists and we skip the file.</li>
+ * <li>AccessDeniedException for excluded files/directories - We skip the file, as files excluded from the analysis, shouldn't throw access exceptions.</li>
+ * </ul>
+ * </p>
+ *
+ * @param file a reference to the file
+ * @param exc the I/O exception that prevented the file from being visited
+ * @throws IOException
+ */
+ @Override
+ public FileVisitResult visitFileFailed(Path file, IOException exc) throws IOException {
+ if (exc instanceof FileSystemLoopException) {
+ LOG.warn("Not indexing due to symlink loop: {}", file.toFile());
+ return FileVisitResult.CONTINUE;
+ } else if (exc instanceof AccessDeniedException && isExcluded(file)) {
+ return FileVisitResult.CONTINUE;
+ }
+ throw exc;
+ }
+
+ /**
+ * <p>Checks if the directory is excluded in the analysis or not. Only the exclusions are checked.</p>
+ *
+ * <p>The inclusions cannot be checked for directories, since the current implementation of pattern matching is intended only for files.</p>
+ *
+ * @param path The file or directory.
+ * @return True if file/directory is excluded from the analysis, false otherwise.
+ */
+ private boolean isExcluded(Path path) throws IOException {
+ Path realAbsoluteFile = path.toRealPath(LinkOption.NOFOLLOW_LINKS).toAbsolutePath().normalize();
+ return isExcludedDirectory(moduleExclusionFilters, realAbsoluteFile, inputModuleHierarchy.root().getBaseDir(), module.getBaseDir(), type);
+ }
+
+ /**
+ * <p>Checks if the path is a directory that is excluded.</p>
+ *
+ * <p>Exclusions patterns are checked both at project and module level.</p>
+ *
+ * @param moduleExclusionFilters The exclusion filters.
+ * @param realAbsoluteFile The path to be checked.
+ * @param projectBaseDir The project base directory.
+ * @param moduleBaseDir The module base directory.
+ * @param type The input file type.
+ * @return True if path is an excluded directory, false otherwise.
+ */
+ private static boolean isExcludedDirectory(ModuleExclusionFilters moduleExclusionFilters, Path realAbsoluteFile, Path projectBaseDir, Path moduleBaseDir,
+ InputFile.Type type) {
+ Path projectRelativePath = projectBaseDir.relativize(realAbsoluteFile);
+ Path moduleRelativePath = moduleBaseDir.relativize(realAbsoluteFile);
+ return moduleExclusionFilters.isExcludedAsParentDirectoryOfExcludedChildren(realAbsoluteFile, projectRelativePath, projectBaseDir, type)
+ || moduleExclusionFilters.isExcludedAsParentDirectoryOfExcludedChildren(realAbsoluteFile, moduleRelativePath, moduleBaseDir, type);
+ }
+
+ @Override
+ public FileVisitResult postVisitDirectory(Path dir, IOException exc) {
+ return FileVisitResult.CONTINUE;
+ }
+
+ private static boolean isHidden(Path path) throws IOException {
+ if (SystemUtils.IS_OS_WINDOWS) {
+ try {
+ DosFileAttributes dosFileAttributes = Files.readAttributes(path, DosFileAttributes.class, LinkOption.NOFOLLOW_LINKS);
+ return dosFileAttributes.isHidden();
+ } catch (UnsupportedOperationException e) {
+ return path.toFile().isHidden();
+ }
+ } else {
+ return Files.isHidden(path);
+ }
+ }
+
+ @FunctionalInterface
+ interface FileVisitAction {
+ void execute(Path file) throws IOException;
+ }
+}
+
*/
package org.sonar.scanner.scan.filesystem;
-import java.io.IOException;
-import java.nio.file.Files;
-import java.nio.file.LinkOption;
import java.nio.file.Path;
import java.util.Arrays;
-import java.util.function.BooleanSupplier;
-import javax.annotation.Nullable;
-import org.apache.commons.io.FilenameUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import org.sonar.api.CoreProperties;
import org.sonar.api.batch.fs.InputFile;
import org.sonar.api.batch.fs.InputFile.Type;
import org.sonar.api.batch.fs.internal.DefaultInputModule;
import org.sonar.api.batch.fs.internal.DefaultInputProject;
import org.sonar.api.batch.fs.internal.SensorStrategy;
-import org.sonar.api.batch.scm.IgnoreCommand;
-import org.sonar.api.notifications.AnalysisWarnings;
import org.sonar.api.utils.MessageException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import org.sonar.scanner.issue.ignore.scanner.IssueExclusionsLoader;
import org.sonar.scanner.repository.language.Language;
import org.sonar.scanner.scan.ScanProperties;
private static final Logger LOG = LoggerFactory.getLogger(FileIndexer.class);
- private final AnalysisWarnings analysisWarnings;
private final ScanProperties properties;
- private final InputFileFilter[] filters;
- private final ProjectExclusionFilters projectExclusionFilters;
private final ProjectCoverageAndDuplicationExclusions projectCoverageAndDuplicationExclusions;
private final IssueExclusionsLoader issueExclusionsLoader;
private final MetadataGenerator metadataGenerator;
private final StatusDetection statusDetection;
private final ScmChangedFiles scmChangedFiles;
- private boolean warnInclusionsAlreadyLogged;
- private boolean warnExclusionsAlreadyLogged;
- private boolean warnCoverageExclusionsAlreadyLogged;
- private boolean warnDuplicationExclusionsAlreadyLogged;
+ private final ModuleRelativePathWarner moduleRelativePathWarner;
+ private final InputFileFilterRepository inputFileFilterRepository;
public FileIndexer(DefaultInputProject project, ScannerComponentIdGenerator scannerComponentIdGenerator, InputComponentStore componentStore,
- ProjectExclusionFilters projectExclusionFilters, ProjectCoverageAndDuplicationExclusions projectCoverageAndDuplicationExclusions, IssueExclusionsLoader issueExclusionsLoader,
- MetadataGenerator metadataGenerator, SensorStrategy sensorStrategy, LanguageDetection languageDetection, AnalysisWarnings analysisWarnings, ScanProperties properties,
- InputFileFilter[] filters, ScmChangedFiles scmChangedFiles, StatusDetection statusDetection) {
+ ProjectCoverageAndDuplicationExclusions projectCoverageAndDuplicationExclusions, IssueExclusionsLoader issueExclusionsLoader,
+ MetadataGenerator metadataGenerator, SensorStrategy sensorStrategy, LanguageDetection languageDetection, ScanProperties properties,
+ ScmChangedFiles scmChangedFiles, StatusDetection statusDetection, ModuleRelativePathWarner moduleRelativePathWarner, InputFileFilterRepository inputFileFilterRepository) {
this.project = project;
this.scannerComponentIdGenerator = scannerComponentIdGenerator;
this.componentStore = componentStore;
this.metadataGenerator = metadataGenerator;
this.sensorStrategy = sensorStrategy;
this.langDetection = languageDetection;
- this.analysisWarnings = analysisWarnings;
this.properties = properties;
- this.filters = filters;
- this.projectExclusionFilters = projectExclusionFilters;
this.scmChangedFiles = scmChangedFiles;
this.statusDetection = statusDetection;
+ this.moduleRelativePathWarner = moduleRelativePathWarner;
+ this.inputFileFilterRepository = inputFileFilterRepository;
}
- void indexFile(DefaultInputModule module, ModuleExclusionFilters moduleExclusionFilters, ModuleCoverageAndDuplicationExclusions moduleCoverageAndDuplicationExclusions,
- Path sourceFile, Type type, ProgressReport progressReport, ProjectFileIndexer.ExclusionCounter exclusionCounter, @Nullable IgnoreCommand ignoreCommand)
- throws IOException {
- // get case of real file without resolving link
- Path realAbsoluteFile = sourceFile.toRealPath(LinkOption.NOFOLLOW_LINKS).toAbsolutePath().normalize();
- Path projectRelativePath = project.getBaseDir().relativize(realAbsoluteFile);
- Path moduleRelativePath = module.getBaseDir().relativize(realAbsoluteFile);
- boolean included = evaluateInclusionsFilters(moduleExclusionFilters, realAbsoluteFile, projectRelativePath, moduleRelativePath, type);
- if (!included) {
- exclusionCounter.increaseByPatternsCount();
- return;
- }
- boolean excluded = evaluateExclusionsFilters(moduleExclusionFilters, realAbsoluteFile, projectRelativePath, moduleRelativePath, type);
- if (excluded) {
- exclusionCounter.increaseByPatternsCount();
- return;
- }
- if (!realAbsoluteFile.startsWith(project.getBaseDir())) {
- LOG.warn("File '{}' is ignored. It is not located in project basedir '{}'.", realAbsoluteFile.toAbsolutePath(), project.getBaseDir());
- return;
- }
- if (!realAbsoluteFile.startsWith(module.getBaseDir())) {
- LOG.warn("File '{}' is ignored. It is not located in module basedir '{}'.", realAbsoluteFile.toAbsolutePath(), module.getBaseDir());
- return;
- }
-
- if (Files.exists(realAbsoluteFile) && isFileSizeBiggerThanLimit(realAbsoluteFile)) {
- LOG.warn("File '{}' is bigger than {}MB and as consequence is removed from the analysis scope.", realAbsoluteFile.toAbsolutePath(), properties.fileSizeLimit());
- return;
- }
-
- Language language = langDetection.language(realAbsoluteFile, projectRelativePath);
+ void indexFile(DefaultInputModule module, ModuleCoverageAndDuplicationExclusions moduleCoverageAndDuplicationExclusions, Path sourceFile,
+ Type type, ProgressReport progressReport) {
+ Path projectRelativePath = project.getBaseDir().relativize(sourceFile);
+ Path moduleRelativePath = module.getBaseDir().relativize(sourceFile);
- if (ignoreCommand != null && ignoreCommand.isIgnored(realAbsoluteFile)) {
- LOG.debug("File '{}' is excluded by the scm ignore settings.", realAbsoluteFile);
- exclusionCounter.increaseByScmCount();
- return;
- }
+ // This should be fast; language should be cached from preprocessing step
+ Language language = langDetection.language(sourceFile, projectRelativePath);
DefaultIndexedFile indexedFile = new DefaultIndexedFile(
- realAbsoluteFile,
+ sourceFile,
project.key(),
projectRelativePath.toString(),
moduleRelativePath.toString(),
language != null ? language.key() : null,
scannerComponentIdGenerator.getAsInt(),
sensorStrategy,
- scmChangedFiles.getOldRelativeFilePath(realAbsoluteFile)
+ scmChangedFiles.getOldRelativeFilePath(sourceFile)
);
DefaultInputFile inputFile = new DefaultInputFile(indexedFile, f -> metadataGenerator.setMetadata(module.key(), f, module.getEncoding()),
componentStore.put(module.key(), inputFile);
issueExclusionsLoader.addMulticriteriaPatterns(inputFile);
String langStr = inputFile.language() != null ? format("with language '%s'", inputFile.language()) : "with no language";
- LOG.debug("'{}' indexed {}{}", projectRelativePath, type == Type.TEST ? "as test " : "", langStr);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("'{}' indexed {}{}", projectRelativePath, type == Type.TEST ? "as test " : "", langStr);
+ }
evaluateCoverageExclusions(moduleCoverageAndDuplicationExclusions, inputFile);
evaluateDuplicationExclusions(moduleCoverageAndDuplicationExclusions, inputFile);
if (properties.preloadFileMetadata()) {
progressReport.message(count + " " + pluralizeFiles(count) + " indexed... (last one was " + inputFile.getProjectRelativePath() + ")");
}
- private boolean evaluateInclusionsFilters(ModuleExclusionFilters moduleExclusionFilters, Path realAbsoluteFile, Path projectRelativePath, Path moduleRelativePath,
- InputFile.Type type) {
- if (!Arrays.equals(moduleExclusionFilters.getInclusionsConfig(type), projectExclusionFilters.getInclusionsConfig(type))) {
- // Module specific configuration
- return moduleExclusionFilters.isIncluded(realAbsoluteFile, moduleRelativePath, type);
- }
- boolean includedByProjectConfiguration = projectExclusionFilters.isIncluded(realAbsoluteFile, projectRelativePath, type);
- if (includedByProjectConfiguration) {
- return true;
- } else if (moduleExclusionFilters.isIncluded(realAbsoluteFile, moduleRelativePath, type)) {
- warnOnce(
- type == Type.MAIN ? CoreProperties.PROJECT_INCLUSIONS_PROPERTY : CoreProperties.PROJECT_TEST_INCLUSIONS_PROPERTY,
- FilenameUtils.normalize(projectRelativePath.toString(), true), () -> warnInclusionsAlreadyLogged, () -> warnInclusionsAlreadyLogged = true);
- return true;
- }
- return false;
- }
-
- private boolean evaluateExclusionsFilters(ModuleExclusionFilters moduleExclusionFilters, Path realAbsoluteFile, Path projectRelativePath, Path moduleRelativePath,
- InputFile.Type type) {
- if (!Arrays.equals(moduleExclusionFilters.getExclusionsConfig(type), projectExclusionFilters.getExclusionsConfig(type))) {
- // Module specific configuration
- return moduleExclusionFilters.isExcluded(realAbsoluteFile, moduleRelativePath, type);
- }
- boolean includedByProjectConfiguration = projectExclusionFilters.isExcluded(realAbsoluteFile, projectRelativePath, type);
- if (includedByProjectConfiguration) {
- return true;
- } else if (moduleExclusionFilters.isExcluded(realAbsoluteFile, moduleRelativePath, type)) {
- warnOnce(
- type == Type.MAIN ? CoreProperties.PROJECT_EXCLUSIONS_PROPERTY : CoreProperties.PROJECT_TEST_EXCLUSIONS_PROPERTY,
- FilenameUtils.normalize(projectRelativePath.toString(), true), () -> warnExclusionsAlreadyLogged, () -> warnExclusionsAlreadyLogged = true);
- return true;
- }
- return false;
- }
-
private void checkIfAlreadyIndexed(DefaultInputFile inputFile) {
if (componentStore.inputFile(inputFile.getProjectRelativePath()) != null) {
throw MessageException.of("File " + inputFile + " can't be indexed twice. Please check that inclusion/exclusion patterns produce "
if (excludedByProjectConfiguration) {
return true;
} else if (moduleCoverageAndDuplicationExclusions.isExcludedForCoverage(inputFile)) {
- warnOnce(CoreProperties.PROJECT_COVERAGE_EXCLUSIONS_PROPERTY, inputFile.getProjectRelativePath(), () -> warnCoverageExclusionsAlreadyLogged,
- () -> warnCoverageExclusionsAlreadyLogged = true);
+ moduleRelativePathWarner.warnOnce(CoreProperties.PROJECT_COVERAGE_EXCLUSIONS_PROPERTY, inputFile.getProjectRelativePath());
return true;
}
return false;
if (excludedByProjectConfiguration) {
return true;
} else if (moduleCoverageAndDuplicationExclusions.isExcludedForDuplication(inputFile)) {
- warnOnce(CoreProperties.CPD_EXCLUSIONS, inputFile.getProjectRelativePath(), () -> warnDuplicationExclusionsAlreadyLogged,
- () -> warnDuplicationExclusionsAlreadyLogged = true);
+ moduleRelativePathWarner.warnOnce(CoreProperties.CPD_EXCLUSIONS, inputFile.getProjectRelativePath());
return true;
}
return false;
}
- private void warnOnce(String propKey, String filePath, BooleanSupplier alreadyLoggedGetter, Runnable markAsLogged) {
- if (!alreadyLoggedGetter.getAsBoolean()) {
- String msg = "Specifying module-relative paths at project level in the property '" + propKey + "' is deprecated. " +
- "To continue matching files like '" + filePath + "', update this property so that patterns refer to project-relative paths.";
- LOG.warn(msg);
- analysisWarnings.addUnique(msg);
- markAsLogged.run();
- }
- }
-
private boolean accept(InputFile indexedFile) {
// InputFileFilter extensions. Might trigger generation of metadata
- for (InputFileFilter filter : filters) {
+ for (InputFileFilter filter : inputFileFilterRepository.getInputFileFilters()) {
if (!filter.accept(indexedFile)) {
LOG.debug("'{}' excluded by {}", indexedFile, filter.getClass().getName());
return false;
return count == 1 ? "file" : "files";
}
- private boolean isFileSizeBiggerThanLimit(Path filePath) throws IOException {
- return Files.size(filePath) > properties.fileSizeLimit() * 1024L * 1024L;
- }
+
}
--- /dev/null
+/*
+ * SonarQube
+ * Copyright (C) 2009-2023 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+package org.sonar.scanner.scan.filesystem;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.LinkOption;
+import java.nio.file.Path;
+import java.util.Arrays;
+import java.util.Optional;
+import javax.annotation.CheckForNull;
+import org.apache.commons.io.FilenameUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.sonar.api.CoreProperties;
+import org.sonar.api.batch.fs.InputFile;
+import org.sonar.api.batch.fs.internal.DefaultInputModule;
+import org.sonar.api.batch.fs.internal.DefaultInputProject;
+import org.sonar.api.batch.scm.IgnoreCommand;
+import org.sonar.scanner.scan.ScanProperties;
+
+public class FilePreprocessor {
+
+ private static final Logger LOG = LoggerFactory.getLogger(FilePreprocessor.class);
+
+ private final ModuleRelativePathWarner moduleRelativePathWarner;
+ private final DefaultInputProject project;
+ private final LanguageDetection languageDetection;
+ private final ProjectExclusionFilters projectExclusionFilters;
+ private final ScanProperties properties;
+
+ public FilePreprocessor(ModuleRelativePathWarner moduleRelativePathWarner, DefaultInputProject project,
+ LanguageDetection languageDetection, ProjectExclusionFilters projectExclusionFilters, ScanProperties properties) {
+ this.moduleRelativePathWarner = moduleRelativePathWarner;
+ this.project = project;
+ this.languageDetection = languageDetection;
+ this.projectExclusionFilters = projectExclusionFilters;
+ this.properties = properties;
+ }
+
+ public Optional<Path> processFile(DefaultInputModule module, ModuleExclusionFilters moduleExclusionFilters, Path sourceFile,
+ InputFile.Type type, ProjectFilePreprocessor.ExclusionCounter exclusionCounter, @CheckForNull IgnoreCommand ignoreCommand) throws IOException {
+ // get case of real file without resolving link
+ Path realAbsoluteFile = sourceFile.toRealPath(LinkOption.NOFOLLOW_LINKS).toAbsolutePath().normalize();
+ Path projectRelativePath = project.getBaseDir().relativize(realAbsoluteFile);
+ Path moduleRelativePath = module.getBaseDir().relativize(realAbsoluteFile);
+ boolean included = isFileIncluded(moduleExclusionFilters, realAbsoluteFile, projectRelativePath, moduleRelativePath, type);
+ if (!included) {
+ exclusionCounter.increaseByPatternsCount();
+ return Optional.empty();
+ }
+ boolean excluded = isFileExcluded(moduleExclusionFilters, realAbsoluteFile, projectRelativePath, moduleRelativePath, type);
+ if (excluded) {
+ exclusionCounter.increaseByPatternsCount();
+ return Optional.empty();
+ }
+
+ if (!realAbsoluteFile.startsWith(project.getBaseDir())) {
+ LOG.warn("File '{}' is ignored. It is not located in project basedir '{}'.", realAbsoluteFile.toAbsolutePath(), project.getBaseDir());
+ return Optional.empty();
+ }
+ if (!realAbsoluteFile.startsWith(module.getBaseDir())) {
+ LOG.warn("File '{}' is ignored. It is not located in module basedir '{}'.", realAbsoluteFile.toAbsolutePath(), module.getBaseDir());
+ return Optional.empty();
+ }
+
+ if (ignoreCommand != null && ignoreCommand.isIgnored(realAbsoluteFile)) {
+ LOG.debug("File '{}' is excluded by the scm ignore settings.", realAbsoluteFile);
+ exclusionCounter.increaseByScmCount();
+ return Optional.empty();
+ }
+
+ if (Files.exists(realAbsoluteFile) && isFileSizeBiggerThanLimit(realAbsoluteFile)) {
+ LOG.warn("File '{}' is bigger than {}MB and as consequence is removed from the analysis scope.", realAbsoluteFile.toAbsolutePath(), properties.fileSizeLimit());
+ return Optional.empty();
+ }
+
+ languageDetection.language(realAbsoluteFile, projectRelativePath);
+
+ return Optional.of(realAbsoluteFile);
+ }
+
+ private boolean isFileIncluded(ModuleExclusionFilters moduleExclusionFilters, Path realAbsoluteFile, Path projectRelativePath,
+ Path moduleRelativePath, InputFile.Type type) {
+ if (!Arrays.equals(moduleExclusionFilters.getInclusionsConfig(type), projectExclusionFilters.getInclusionsConfig(type))) {
+ return moduleExclusionFilters.isIncluded(realAbsoluteFile, moduleRelativePath, type);
+ }
+ boolean includedByProjectConfiguration = projectExclusionFilters.isIncluded(realAbsoluteFile, projectRelativePath, type);
+ if (includedByProjectConfiguration) {
+ return true;
+ }
+ if (moduleExclusionFilters.isIncluded(realAbsoluteFile, moduleRelativePath, type)) {
+ moduleRelativePathWarner.warnOnce(
+ type == InputFile.Type.MAIN ? CoreProperties.PROJECT_INCLUSIONS_PROPERTY : CoreProperties.PROJECT_TEST_INCLUSIONS_PROPERTY,
+ FilenameUtils.normalize(projectRelativePath.toString(), true));
+ return true;
+ }
+ return false;
+ }
+
+ private boolean isFileExcluded(ModuleExclusionFilters moduleExclusionFilters, Path realAbsoluteFile, Path projectRelativePath,
+ Path moduleRelativePath, InputFile.Type type) {
+ if (!Arrays.equals(moduleExclusionFilters.getExclusionsConfig(type), projectExclusionFilters.getExclusionsConfig(type))) {
+ return moduleExclusionFilters.isExcluded(realAbsoluteFile, moduleRelativePath, type);
+ }
+ boolean includedByProjectConfiguration = projectExclusionFilters.isExcluded(realAbsoluteFile, projectRelativePath, type);
+ if (includedByProjectConfiguration) {
+ return true;
+ }
+ if (moduleExclusionFilters.isExcluded(realAbsoluteFile, moduleRelativePath, type)) {
+ moduleRelativePathWarner.warnOnce(
+ type == InputFile.Type.MAIN ? CoreProperties.PROJECT_EXCLUSIONS_PROPERTY : CoreProperties.PROJECT_TEST_EXCLUSIONS_PROPERTY,
+ FilenameUtils.normalize(projectRelativePath.toString(), true));
+ return true;
+ }
+ return false;
+ }
+
+ private boolean isFileSizeBiggerThanLimit(Path filePath) throws IOException {
+ return Files.size(filePath) > properties.fileSizeLimit() * 1024L * 1024L;
+ }
+}
--- /dev/null
+/*
+ * SonarQube
+ * Copyright (C) 2009-2023 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+package org.sonar.scanner.scan.filesystem;
+
+import org.sonar.api.batch.fs.InputFileFilter;
+
+public class InputFileFilterRepository {
+ private final InputFileFilter[] inputFileFilters;
+
+ public InputFileFilterRepository(InputFileFilter... inputFileFilters) {
+ this.inputFileFilters = inputFileFilters;
+ }
+
+ public InputFileFilter[] getInputFileFilters() {
+ return inputFileFilters;
+ }
+}
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import javax.annotation.CheckForNull;
*/
private final Map<Language, PathPattern[]> patternsByLanguage;
private final List<Language> languagesToConsider;
+ private final Map<String, Language> languageCacheByPath;
- public LanguageDetection(Configuration settings, LanguagesRepository languages) {
+ public LanguageDetection(Configuration settings, LanguagesRepository languages, Map<String, Language> languageCache) {
Map<Language, PathPattern[]> patternsByLanguageBuilder = new LinkedHashMap<>();
for (Language language : languages.all()) {
String[] filePatterns = settings.getStringArray(getFileLangPatternPropKey(language.key()));
languagesToConsider = List.copyOf(patternsByLanguageBuilder.keySet());
patternsByLanguage = unmodifiableMap(patternsByLanguageBuilder);
+ languageCacheByPath = languageCache;
}
private static PathPattern[] getLanguagePatterns(Language language) {
@CheckForNull
Language language(Path absolutePath, Path relativePath) {
- Language detectedLanguage = null;
+ Language detectedLanguage = languageCacheByPath.get(absolutePath.toString());
+ if (detectedLanguage != null) {
+ return detectedLanguage;
+ }
+
for (Language language : languagesToConsider) {
if (isCandidateForLanguage(absolutePath, relativePath, language)) {
if (detectedLanguage == null) {
detectedLanguage = language;
+ languageCacheByPath.put(absolutePath.toString(), language);
} else {
// Language was already forced by another pattern
throw MessageException.of(MessageFormat.format("Language of file ''{0}'' can not be decided as the file matches patterns of both {1} and {2}",
return detectedLanguage;
}
+ public Set<String> getDetectedLanguages() {
+ return languageCacheByPath.values().stream().map(Language::key).collect(Collectors.toSet());
+ }
+
private boolean isCandidateForLanguage(Path absolutePath, Path relativePath, Language language) {
PathPattern[] patterns = patternsByLanguage.get(language);
return patterns != null && Arrays.stream(patterns).anyMatch(pattern -> pattern.match(absolutePath, relativePath, false));
--- /dev/null
+/*
+ * SonarQube
+ * Copyright (C) 2009-2023 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+package org.sonar.scanner.scan.filesystem;
+
+import java.util.HashSet;
+import java.util.Set;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.sonar.api.notifications.AnalysisWarnings;
+
+public class ModuleRelativePathWarner {
+
+ private static final Logger LOG = LoggerFactory.getLogger(ModuleRelativePathWarner.class);
+ private final AnalysisWarnings analysisWarnings;
+ private final Set<String> previouslyWarnedProps = new HashSet<>();
+
+ public ModuleRelativePathWarner(AnalysisWarnings analysisWarnings) {
+ this.analysisWarnings = analysisWarnings;
+ }
+
+ public void warnOnce(String propKey, String filePath) {
+ if (!previouslyWarnedProps.contains(propKey)) {
+ previouslyWarnedProps.add(propKey);
+ String msg = "Specifying module-relative paths at project level in the property '" + propKey + "' is deprecated. " +
+ "To continue matching files like '" + filePath + "', update this property so that patterns refer to project-relative paths.";
+ LOG.warn(msg);
+ analysisWarnings.addUnique(msg);
+ }
+ }
+}
package org.sonar.scanner.scan.filesystem;
import java.io.IOException;
-import java.nio.file.AccessDeniedException;
-import java.nio.file.FileSystemLoopException;
import java.nio.file.FileVisitOption;
-import java.nio.file.FileVisitResult;
-import java.nio.file.FileVisitor;
import java.nio.file.Files;
-import java.nio.file.LinkOption;
import java.nio.file.Path;
-import java.nio.file.attribute.BasicFileAttributes;
-import java.nio.file.attribute.DosFileAttributes;
import java.util.Collections;
import java.util.Comparator;
import java.util.Iterator;
import java.util.List;
import java.util.Optional;
import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicInteger;
import org.apache.commons.lang.StringUtils;
-import org.apache.commons.lang.SystemUtils;
-import org.sonar.api.batch.fs.InputFile;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import org.sonar.api.batch.fs.InputFile.Type;
import org.sonar.api.batch.fs.internal.DefaultInputModule;
-import org.sonar.api.batch.scm.IgnoreCommand;
import org.sonar.api.notifications.AnalysisWarnings;
import org.sonar.api.scan.filesystem.PathResolver;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import org.sonar.scanner.bootstrap.GlobalConfiguration;
import org.sonar.scanner.bootstrap.GlobalServerSettings;
import org.sonar.scanner.fs.InputModuleHierarchy;
import org.sonar.scanner.scan.ModuleConfigurationProvider;
import org.sonar.scanner.scan.ProjectServerSettings;
import org.sonar.scanner.scan.SonarGlobalPropertiesFilter;
-import org.sonar.scanner.scm.ScmConfiguration;
import org.sonar.scanner.util.ProgressReport;
-import static java.util.Collections.emptyList;
-import static java.util.Collections.singletonList;
-
/**
* Index project input files into {@link InputComponentStore}.
*/
private final ProjectExclusionFilters projectExclusionFilters;
private final SonarGlobalPropertiesFilter sonarGlobalPropertiesFilter;
private final ProjectCoverageAndDuplicationExclusions projectCoverageAndDuplicationExclusions;
- private final ScmConfiguration scmConfiguration;
private final InputComponentStore componentStore;
private final InputModuleHierarchy inputModuleHierarchy;
private final GlobalConfiguration globalConfig;
private final GlobalServerSettings globalServerSettings;
private final ProjectServerSettings projectServerSettings;
private final FileIndexer fileIndexer;
- private final IgnoreCommand ignoreCommand;
- private final boolean useScmExclusion;
+ private final ProjectFilePreprocessor projectFilePreprocessor;
private final AnalysisWarnings analysisWarnings;
private ProgressReport progressReport;
public ProjectFileIndexer(InputComponentStore componentStore, ProjectExclusionFilters exclusionFilters,
SonarGlobalPropertiesFilter sonarGlobalPropertiesFilter, InputModuleHierarchy inputModuleHierarchy,
GlobalConfiguration globalConfig, GlobalServerSettings globalServerSettings, ProjectServerSettings projectServerSettings,
- FileIndexer fileIndexer, ProjectCoverageAndDuplicationExclusions projectCoverageAndDuplicationExclusions, ScmConfiguration scmConfiguration,
- AnalysisWarnings analysisWarnings) {
+ FileIndexer fileIndexer, ProjectCoverageAndDuplicationExclusions projectCoverageAndDuplicationExclusions,
+ ProjectFilePreprocessor projectFilePreprocessor, AnalysisWarnings analysisWarnings) {
this.componentStore = componentStore;
this.sonarGlobalPropertiesFilter = sonarGlobalPropertiesFilter;
this.inputModuleHierarchy = inputModuleHierarchy;
this.fileIndexer = fileIndexer;
this.projectExclusionFilters = exclusionFilters;
this.projectCoverageAndDuplicationExclusions = projectCoverageAndDuplicationExclusions;
- this.scmConfiguration = scmConfiguration;
+ this.projectFilePreprocessor = projectFilePreprocessor;
this.analysisWarnings = analysisWarnings;
- this.ignoreCommand = loadIgnoreCommand();
- this.useScmExclusion = ignoreCommand != null;
}
public void index() {
LOG.info("Project configuration:");
projectExclusionFilters.log(" ");
projectCoverageAndDuplicationExclusions.log(" ");
- ExclusionCounter exclusionCounter = new ExclusionCounter();
- if (useScmExclusion) {
- ignoreCommand.init(inputModuleHierarchy.root().getBaseDir().toAbsolutePath());
- indexModulesRecursively(inputModuleHierarchy.root(), exclusionCounter);
- ignoreCommand.clean();
- } else {
- indexModulesRecursively(inputModuleHierarchy.root(), exclusionCounter);
- }
+ indexModulesRecursively(inputModuleHierarchy.root());
int totalIndexed = componentStore.inputFiles().size();
progressReport.stop(totalIndexed + " " + pluralizeFiles(totalIndexed) + " indexed");
- int excludedFileByPatternCount = exclusionCounter.getByPatternsCount();
- if (projectExclusionFilters.hasPattern() || excludedFileByPatternCount > 0) {
- LOG.info("{} {} ignored because of inclusion/exclusion patterns", excludedFileByPatternCount, pluralizeFiles(excludedFileByPatternCount));
- }
- int excludedFileByScmCount = exclusionCounter.getByScmCount();
- if (useScmExclusion) {
- LOG.info("{} {} ignored because of scm ignore settings", excludedFileByScmCount, pluralizeFiles(excludedFileByScmCount));
- }
}
- private IgnoreCommand loadIgnoreCommand() {
- try {
- if (!scmConfiguration.isExclusionDisabled() && scmConfiguration.provider() != null) {
- return scmConfiguration.provider().ignoreCommand();
- }
- } catch (UnsupportedOperationException e) {
- LOG.debug("File exclusion based on SCM ignore information is not available with this plugin.");
- }
-
- return null;
+ private void indexModulesRecursively(DefaultInputModule module) {
+ inputModuleHierarchy.children(module).stream()
+ .sorted(Comparator.comparing(DefaultInputModule::key))
+ .forEach(this::indexModulesRecursively);
+ index(module);
}
- private void indexModulesRecursively(DefaultInputModule module, ExclusionCounter exclusionCounter) {
- inputModuleHierarchy.children(module).stream().sorted(Comparator.comparing(DefaultInputModule::key)).forEach(m -> indexModulesRecursively(m, exclusionCounter));
- index(module, exclusionCounter);
- }
-
- private void index(DefaultInputModule module, ExclusionCounter exclusionCounter) {
+ private void index(DefaultInputModule module) {
// Emulate creation of module level settings
ModuleConfiguration moduleConfig = new ModuleConfigurationProvider(sonarGlobalPropertiesFilter).provide(globalConfig, module, globalServerSettings, projectServerSettings);
ModuleExclusionFilters moduleExclusionFilters = new ModuleExclusionFilters(moduleConfig, analysisWarnings);
moduleExclusionFilters.log(" ");
moduleCoverageAndDuplicationExclusions.log(" ");
}
- boolean hasChildModules = !module.definition().getSubProjects().isEmpty();
- boolean hasTests = module.getTestDirsOrFiles().isPresent();
- // Default to index basedir when no sources provided
- List<Path> mainSourceDirsOrFiles = module.getSourceDirsOrFiles()
- .orElseGet(() -> hasChildModules || hasTests ? emptyList() : singletonList(module.getBaseDir().toAbsolutePath()));
- indexFiles(module, moduleExclusionFilters, moduleCoverageAndDuplicationExclusions, mainSourceDirsOrFiles, Type.MAIN, exclusionCounter);
- module.getTestDirsOrFiles().ifPresent(tests -> indexFiles(module, moduleExclusionFilters, moduleCoverageAndDuplicationExclusions, tests, Type.TEST, exclusionCounter));
+ List<Path> mainSourceDirsOrFiles = projectFilePreprocessor.getMainSourcesByModule(module);
+ indexFiles(module, moduleExclusionFilters, moduleCoverageAndDuplicationExclusions, mainSourceDirsOrFiles, Type.MAIN);
+ projectFilePreprocessor.getTestSourcesByModule(module)
+ .ifPresent(tests -> indexFiles(module, moduleExclusionFilters, moduleCoverageAndDuplicationExclusions, tests, Type.TEST));
}
private static void logPaths(String label, Path baseDir, List<Path> paths) {
for (Iterator<Path> it = paths.iterator(); it.hasNext(); ) {
Path file = it.next();
Optional<String> relativePathToBaseDir = PathResolver.relativize(baseDir, file);
- if (!relativePathToBaseDir.isPresent()) {
+ if (relativePathToBaseDir.isEmpty()) {
sb.append(file);
} else if (StringUtils.isBlank(relativePathToBaseDir.get())) {
sb.append(".");
}
}
- private static String pluralizeFiles(int count) {
- return count == 1 ? "file" : "files";
- }
-
- private void indexFiles(DefaultInputModule module, ModuleExclusionFilters moduleExclusionFilters,
- ModuleCoverageAndDuplicationExclusions moduleCoverageAndDuplicationExclusions, List<Path> sources, Type type, ExclusionCounter exclusionCounter) {
+ private void indexFiles(DefaultInputModule module, ModuleExclusionFilters moduleExclusionFilters, ModuleCoverageAndDuplicationExclusions moduleCoverageAndDuplicationExclusions,
+ List<Path> sources, Type type) {
try {
for (Path dirOrFile : sources) {
if (dirOrFile.toFile().isDirectory()) {
- indexDirectory(module, moduleExclusionFilters, moduleCoverageAndDuplicationExclusions, dirOrFile, type, exclusionCounter);
+ indexDirectory(module, moduleExclusionFilters, moduleCoverageAndDuplicationExclusions, dirOrFile, type);
} else {
- fileIndexer.indexFile(module, moduleExclusionFilters, moduleCoverageAndDuplicationExclusions, dirOrFile, type, progressReport, exclusionCounter,
- ignoreCommand);
+ fileIndexer.indexFile(module, moduleCoverageAndDuplicationExclusions, dirOrFile, type, progressReport);
}
}
} catch (IOException e) {
}
private void indexDirectory(DefaultInputModule module, ModuleExclusionFilters moduleExclusionFilters,
- ModuleCoverageAndDuplicationExclusions moduleCoverageAndDuplicationExclusions, Path dirToIndex, Type type, ExclusionCounter exclusionCounter)
- throws IOException {
+ ModuleCoverageAndDuplicationExclusions moduleCoverageAndDuplicationExclusions,
+ Path dirToIndex, Type type) throws IOException {
Files.walkFileTree(dirToIndex.normalize(), Collections.singleton(FileVisitOption.FOLLOW_LINKS), Integer.MAX_VALUE,
- new IndexFileVisitor(module, moduleExclusionFilters, moduleCoverageAndDuplicationExclusions, type, exclusionCounter));
- }
-
-
- /**
- * <p>Checks if the path is a directory that is excluded.</p>
- *
- * <p>Exclusions patterns are checked both at project and module level.</p>
- *
- * @param moduleExclusionFilters The exclusion filters.
- * @param realAbsoluteFile The path to be checked.
- * @param projectBaseDir The project base directory.
- * @param moduleBaseDir The module base directory.
- * @param type The input file type.
- * @return True if path is an excluded directory, false otherwise.
- */
- private static boolean isExcludedDirectory(ModuleExclusionFilters moduleExclusionFilters, Path realAbsoluteFile, Path projectBaseDir, Path moduleBaseDir,
- InputFile.Type type) {
- Path projectRelativePath = projectBaseDir.relativize(realAbsoluteFile);
- Path moduleRelativePath = moduleBaseDir.relativize(realAbsoluteFile);
- return moduleExclusionFilters.isExcludedAsParentDirectoryOfExcludedChildren(realAbsoluteFile, projectRelativePath, projectBaseDir, type)
- || moduleExclusionFilters.isExcludedAsParentDirectoryOfExcludedChildren(realAbsoluteFile, moduleRelativePath, moduleBaseDir, type);
+ new DirectoryFileVisitor(file -> fileIndexer.indexFile(module, moduleCoverageAndDuplicationExclusions, file, type, progressReport),
+ module, moduleExclusionFilters, inputModuleHierarchy, type));
}
- private class IndexFileVisitor implements FileVisitor<Path> {
- private final DefaultInputModule module;
- private final ModuleExclusionFilters moduleExclusionFilters;
- private final ModuleCoverageAndDuplicationExclusions moduleCoverageAndDuplicationExclusions;
- private final Type type;
- private final ExclusionCounter exclusionCounter;
-
- IndexFileVisitor(DefaultInputModule module, ModuleExclusionFilters moduleExclusionFilters, ModuleCoverageAndDuplicationExclusions moduleCoverageAndDuplicationExclusions,
- Type type,
- ExclusionCounter exclusionCounter) {
- this.module = module;
- this.moduleExclusionFilters = moduleExclusionFilters;
- this.moduleCoverageAndDuplicationExclusions = moduleCoverageAndDuplicationExclusions;
- this.type = type;
- this.exclusionCounter = exclusionCounter;
- }
-
- @Override
- public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException {
- if (isHidden(dir)) {
- return FileVisitResult.SKIP_SUBTREE;
- }
- return FileVisitResult.CONTINUE;
- }
-
- @Override
- public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
- if (!Files.isHidden(file)) {
- fileIndexer.indexFile(module, moduleExclusionFilters, moduleCoverageAndDuplicationExclusions, file, type, progressReport, exclusionCounter, ignoreCommand);
- }
- return FileVisitResult.CONTINUE;
- }
-
- /**
- * <p>Overridden method to handle exceptions while visiting files in the analysis.</p>
- *
- * <p>
- * <ul>
- * <li>FileSystemLoopException - We show a warning that a symlink loop exists and we skip the file.</li>
- * <li>AccessDeniedException for excluded files/directories - We skip the file, as files excluded from the analysis, shouldn't throw access exceptions.</li>
- * </ul>
- * </p>
- *
- * @param file a reference to the file
- * @param exc the I/O exception that prevented the file from being visited
- * @throws IOException
- */
- @Override
- public FileVisitResult visitFileFailed(Path file, IOException exc) throws IOException {
- if (exc instanceof FileSystemLoopException) {
- LOG.warn("Not indexing due to symlink loop: {}", file.toFile());
- return FileVisitResult.CONTINUE;
- } else if (exc instanceof AccessDeniedException && isExcluded(file)) {
- return FileVisitResult.CONTINUE;
- }
- throw exc;
- }
-
- /**
- * <p>Checks if the directory is excluded in the analysis or not. Only the exclusions are checked.</p>
- *
- * <p>The inclusions cannot be checked for directories, since the current implementation of pattern matching is intended only for files.</p>
- *
- * @param path The file or directory.
- * @return True if file/directory is excluded from the analysis, false otherwise.
- */
- private boolean isExcluded(Path path) throws IOException {
- Path realAbsoluteFile = path.toRealPath(LinkOption.NOFOLLOW_LINKS).toAbsolutePath().normalize();
- return isExcludedDirectory(moduleExclusionFilters, realAbsoluteFile, inputModuleHierarchy.root().getBaseDir(), module.getBaseDir(), type);
- }
-
- @Override
- public FileVisitResult postVisitDirectory(Path dir, IOException exc) {
- return FileVisitResult.CONTINUE;
- }
-
- private boolean isHidden(Path path) throws IOException {
- if (SystemUtils.IS_OS_WINDOWS) {
- try {
- DosFileAttributes dosFileAttributes = Files.readAttributes(path, DosFileAttributes.class, LinkOption.NOFOLLOW_LINKS);
- return dosFileAttributes.isHidden();
- } catch (UnsupportedOperationException e) {
- return path.toFile().isHidden();
- }
- } else {
- return Files.isHidden(path);
- }
- }
+ private static String pluralizeFiles(int count) {
+ return count == 1 ? "file" : "files";
}
- static class ExclusionCounter {
- private final AtomicInteger excludedByPatternsCount = new AtomicInteger(0);
- private final AtomicInteger excludedByScmCount = new AtomicInteger(0);
-
- public void increaseByPatternsCount() {
- excludedByPatternsCount.incrementAndGet();
- }
-
- public int getByPatternsCount() {
- return excludedByPatternsCount.get();
- }
- public void increaseByScmCount() {
- excludedByScmCount.incrementAndGet();
- }
- public int getByScmCount() {
- return excludedByScmCount.get();
- }
- }
}
--- /dev/null
+/*
+ * SonarQube
+ * Copyright (C) 2009-2023 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+package org.sonar.scanner.scan.filesystem;
+
+import java.io.IOException;
+import java.nio.file.FileVisitOption;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.concurrent.TimeUnit;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.sonar.api.batch.fs.InputFile;
+import org.sonar.api.batch.fs.internal.DefaultInputModule;
+import org.sonar.api.batch.scm.IgnoreCommand;
+import org.sonar.api.batch.scm.ScmProvider;
+import org.sonar.api.notifications.AnalysisWarnings;
+import org.sonar.scanner.bootstrap.GlobalConfiguration;
+import org.sonar.scanner.bootstrap.GlobalServerSettings;
+import org.sonar.scanner.fs.InputModuleHierarchy;
+import org.sonar.scanner.scan.ModuleConfiguration;
+import org.sonar.scanner.scan.ModuleConfigurationProvider;
+import org.sonar.scanner.scan.ProjectServerSettings;
+import org.sonar.scanner.scan.SonarGlobalPropertiesFilter;
+import org.sonar.scanner.scm.ScmConfiguration;
+import org.sonar.scanner.util.ProgressReport;
+
+import static java.util.Collections.emptyList;
+import static java.util.Collections.singletonList;
+
+public class ProjectFilePreprocessor {
+
+ private static final Logger LOG = LoggerFactory.getLogger(ProjectFilePreprocessor.class);
+ private static final String TELEMETRY_STEP_NAME = "file.preprocessing";
+
+ private final AnalysisWarnings analysisWarnings;
+ private final IgnoreCommand ignoreCommand;
+ private final boolean useScmExclusion;
+ private final ScmConfiguration scmConfiguration;
+ private final InputModuleHierarchy inputModuleHierarchy;
+ private final GlobalConfiguration globalConfig;
+ private final GlobalServerSettings globalServerSettings;
+ private final ProjectServerSettings projectServerSettings;
+ private final LanguageDetection languageDetection;
+ private final FilePreprocessor filePreprocessor;
+ private final ProjectExclusionFilters projectExclusionFilters;
+
+ private final SonarGlobalPropertiesFilter sonarGlobalPropertiesFilter;
+
+ private final Map<DefaultInputModule, List<Path>> mainSourcesByModule = new HashMap<>();
+ private final Map<DefaultInputModule, List<Path>> testSourcesByModule = new HashMap<>();
+
+ private int totalFilesPreprocessed = 0;
+
+ public ProjectFilePreprocessor(AnalysisWarnings analysisWarnings, ScmConfiguration scmConfiguration, InputModuleHierarchy inputModuleHierarchy,
+ GlobalConfiguration globalConfig, GlobalServerSettings globalServerSettings, ProjectServerSettings projectServerSettings,
+ LanguageDetection languageDetection, FilePreprocessor filePreprocessor,
+ ProjectExclusionFilters projectExclusionFilters, SonarGlobalPropertiesFilter sonarGlobalPropertiesFilter) {
+ this.analysisWarnings = analysisWarnings;
+ this.scmConfiguration = scmConfiguration;
+ this.inputModuleHierarchy = inputModuleHierarchy;
+ this.globalConfig = globalConfig;
+ this.globalServerSettings = globalServerSettings;
+ this.projectServerSettings = projectServerSettings;
+ this.languageDetection = languageDetection;
+ this.filePreprocessor = filePreprocessor;
+ this.projectExclusionFilters = projectExclusionFilters;
+ this.sonarGlobalPropertiesFilter = sonarGlobalPropertiesFilter;
+ this.ignoreCommand = loadIgnoreCommand();
+ this.useScmExclusion = ignoreCommand != null;
+ }
+
+ public void execute() {
+ ProgressReport progressReport = new ProgressReport("Report about progress of file preprocessing",
+ TimeUnit.SECONDS.toMillis(10));
+ progressReport.start("Preprocessing files...");
+ ExclusionCounter exclusionCounter = new ExclusionCounter();
+
+ if (useScmExclusion) {
+ ignoreCommand.init(inputModuleHierarchy.root().getBaseDir().toAbsolutePath());
+ processModulesRecursively(inputModuleHierarchy.root(), exclusionCounter);
+ ignoreCommand.clean();
+ } else {
+ processModulesRecursively(inputModuleHierarchy.root(), exclusionCounter);
+ }
+
+ int totalLanguagesDetected = languageDetection.getDetectedLanguages().size();
+
+ progressReport.stop(String.format("%s detected in %s", pluralizeWithCount("language", totalLanguagesDetected),
+ pluralizeWithCount("preprocessed file", totalFilesPreprocessed)));
+
+ int excludedFileByPatternCount = exclusionCounter.getByPatternsCount();
+ if (projectExclusionFilters.hasPattern() || excludedFileByPatternCount > 0) {
+ if (LOG.isInfoEnabled()) {
+ LOG.info("{} ignored because of inclusion/exclusion patterns", pluralizeWithCount("file", excludedFileByPatternCount));
+ }
+ }
+
+ int excludedFileByScmCount = exclusionCounter.getByScmCount();
+ if (useScmExclusion) {
+ if (LOG.isInfoEnabled()) {
+ LOG.info("{} ignored because of scm ignore settings", pluralizeWithCount("file", excludedFileByScmCount));
+ }
+ }
+ }
+
+ private void processModulesRecursively(DefaultInputModule module, ExclusionCounter exclusionCounter) {
+ inputModuleHierarchy.children(module).stream().sorted(Comparator.comparing(DefaultInputModule::key)).forEach(
+ m -> processModulesRecursively(m, exclusionCounter));
+ processModule(module, exclusionCounter);
+ }
+
+ private void processModule(DefaultInputModule module, ExclusionCounter exclusionCounter) {
+ // Emulate creation of module level settings
+ ModuleConfiguration moduleConfig = new ModuleConfigurationProvider(sonarGlobalPropertiesFilter).provide(globalConfig, module, globalServerSettings, projectServerSettings);
+ ModuleExclusionFilters moduleExclusionFilters = new ModuleExclusionFilters(moduleConfig, analysisWarnings);
+ boolean hasChildModules = !module.definition().getSubProjects().isEmpty();
+ boolean hasTests = module.getTestDirsOrFiles().isPresent();
+ // Default to index basedir when no sources provided
+ List<Path> mainSourceDirsOrFiles = module.getSourceDirsOrFiles()
+ .orElseGet(() -> hasChildModules || hasTests ? emptyList() : singletonList(module.getBaseDir().toAbsolutePath()));
+ List<Path> processedSources = processModuleSources(module, moduleExclusionFilters, mainSourceDirsOrFiles, InputFile.Type.MAIN,
+ exclusionCounter);
+ mainSourcesByModule.put(module, processedSources);
+ totalFilesPreprocessed += processedSources.size();
+ module.getTestDirsOrFiles().ifPresent(tests -> {
+ List<Path> processedTestSources = processModuleSources(module, moduleExclusionFilters, tests, InputFile.Type.TEST, exclusionCounter);
+ testSourcesByModule.put(module, processedTestSources);
+ totalFilesPreprocessed += processedTestSources.size();
+ });
+ }
+
+ private List<Path> processModuleSources(DefaultInputModule module, ModuleExclusionFilters moduleExclusionFilters, List<Path> sources,
+ InputFile.Type type, ExclusionCounter exclusionCounter) {
+ List<Path> processedFiles = new ArrayList<>();
+ try {
+ for (Path dirOrFile : sources) {
+ if (dirOrFile.toFile().isDirectory()) {
+ processedFiles.addAll(processDirectory(module, moduleExclusionFilters, dirOrFile, type, exclusionCounter));
+ } else {
+ filePreprocessor.processFile(module, moduleExclusionFilters, dirOrFile, type, exclusionCounter, ignoreCommand)
+ .ifPresent(processedFiles::add);
+ }
+ }
+ } catch (IOException e) {
+ throw new IllegalStateException("Failed to preprocess files", e);
+ }
+ return processedFiles;
+ }
+
+ private List<Path> processDirectory(DefaultInputModule module, ModuleExclusionFilters moduleExclusionFilters, Path path,
+ InputFile.Type type, ExclusionCounter exclusionCounter) throws IOException {
+ List<Path> processedFiles = new ArrayList<>();
+ Files.walkFileTree(path.normalize(), Collections.singleton(FileVisitOption.FOLLOW_LINKS), Integer.MAX_VALUE,
+ new DirectoryFileVisitor(file -> filePreprocessor.processFile(module, moduleExclusionFilters, file, type, exclusionCounter,
+ ignoreCommand).ifPresent(processedFiles::add), module, moduleExclusionFilters, inputModuleHierarchy, type)
+ );
+ return processedFiles;
+ }
+
+ public List<Path> getMainSourcesByModule(DefaultInputModule module) {
+ return Collections.unmodifiableList(mainSourcesByModule.get(module));
+ }
+
+ public Optional<List<Path>> getTestSourcesByModule(DefaultInputModule module) {
+ return Optional.ofNullable(testSourcesByModule.get(module)).map(Collections::unmodifiableList);
+ }
+
+ private IgnoreCommand loadIgnoreCommand() {
+ try {
+ ScmProvider provider = scmConfiguration.provider();
+ if (!scmConfiguration.isExclusionDisabled() && provider != null) {
+ return provider.ignoreCommand();
+ }
+ } catch (UnsupportedOperationException e) {
+ LOG.debug("File exclusion based on SCM ignore information is not available with this plugin.");
+ }
+
+ return null;
+ }
+
+ private static String pluralizeWithCount(String str, int count) {
+ String pluralized = count == 1 ? str : (str + "s");
+ return count + " " + pluralized;
+ }
+
+ public static class ExclusionCounter {
+ private int excludedByPatternsCount = 0;
+ private int excludedByScmCount = 0;
+
+ public void increaseByPatternsCount() {
+ excludedByPatternsCount++;
+ }
+
+ public int getByPatternsCount() {
+ return excludedByPatternsCount;
+ }
+
+ public void increaseByScmCount() {
+ excludedByScmCount++;
+ }
+
+ public int getByScmCount() {
+ return excludedByScmCount;
+ }
+ }
+}
--- /dev/null
+/*
+ * SonarQube
+ * Copyright (C) 2009-2023 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+package org.sonar.scanner.scan.filesystem;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.FileSystemLoopException;
+import java.nio.file.FileVisitResult;
+import java.nio.file.Files;
+import java.nio.file.LinkOption;
+import java.nio.file.Path;
+import java.nio.file.attribute.BasicFileAttributes;
+import org.apache.commons.lang.SystemUtils;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.sonar.api.batch.fs.InputFile;
+import org.sonar.api.batch.fs.internal.DefaultInputModule;
+import org.sonar.scanner.fs.InputModuleHierarchy;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.Assert.assertThrows;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.verify;
+
+public class DirectoryFileVisitorTest {
+
+ @ClassRule
+ public static TemporaryFolder temp = new TemporaryFolder();
+
+ private final DefaultInputModule module = mock();
+ private final ModuleExclusionFilters moduleExclusionFilters = mock();
+ private final InputModuleHierarchy inputModuleHierarchy = mock();
+ private final InputFile.Type type = mock();
+
+ @Test
+ public void visit_hidden_file() throws IOException {
+ DirectoryFileVisitor.FileVisitAction action = mock(DirectoryFileVisitor.FileVisitAction.class);
+
+ File hidden = temp.newFile(".hidden");
+ if (SystemUtils.IS_OS_WINDOWS) {
+ Files.setAttribute(hidden.toPath(), "dos:hidden", true, LinkOption.NOFOLLOW_LINKS);
+ }
+
+
+ DirectoryFileVisitor underTest = new DirectoryFileVisitor(action, module, moduleExclusionFilters, inputModuleHierarchy, type);
+ underTest.visitFile(hidden.toPath(), Files.readAttributes(hidden.toPath(), BasicFileAttributes.class));
+
+ verify(action, never()).execute(any(Path.class));
+ }
+
+ @Test
+ public void test_visit_file_failed_generic_io_exception() throws IOException {
+ DirectoryFileVisitor.FileVisitAction action = mock(DirectoryFileVisitor.FileVisitAction.class);
+
+ File file = temp.newFile("failed");
+
+ DirectoryFileVisitor underTest = new DirectoryFileVisitor(action, module, moduleExclusionFilters, inputModuleHierarchy, type);
+ assertThrows(IOException.class, () -> underTest.visitFileFailed(file.toPath(), new IOException()));
+ }
+
+ @Test
+ public void test_visit_file_failed_file_system_loop_exception() throws IOException {
+ DirectoryFileVisitor.FileVisitAction action = mock(DirectoryFileVisitor.FileVisitAction.class);
+
+ File file = temp.newFile("symlink");
+
+ DirectoryFileVisitor underTest = new DirectoryFileVisitor(action, module, moduleExclusionFilters, inputModuleHierarchy, type);
+ FileVisitResult result = underTest.visitFileFailed(file.toPath(), new FileSystemLoopException(file.getPath()));
+
+ assertThat(result).isEqualTo(FileVisitResult.CONTINUE);
+ }
+
+}
--- /dev/null
+/*
+ * SonarQube
+ * Copyright (C) 2009-2023 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+package org.sonar.scanner.scan.filesystem;
+
+import org.junit.Test;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+public class InputFileFilterRepositoryTest {
+
+ @Test
+ public void should_not_return_null_if_initialized_with_no_filters() {
+ InputFileFilterRepository underTest = new InputFileFilterRepository();
+ assertThat(underTest.getInputFileFilters()).isNotNull();
+ }
+
+ @Test
+ public void should_return_filters_from_initialization() {
+ InputFileFilterRepository underTest = new InputFileFilterRepository(f -> true);
+ assertThat(underTest.getInputFileFilters()).isNotNull();
+ assertThat(underTest.getInputFileFilters()).hasSize(1);
+ }
+
+}
import com.tngtech.java.junit.dataprovider.UseDataProvider;
import java.io.File;
import java.nio.file.Paths;
+import java.util.HashMap;
+import java.util.Map;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.endsWith;
import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
@RunWith(DataProviderRunner.class)
public class LanguageDetectionTest {
@Test
public void detectLanguageKey_shouldDetectByFileExtension() {
LanguagesRepository languages = new FakeLanguagesRepository(new Languages(new MockLanguage("java", "java", "jav"), new MockLanguage("cobol", "cbl", "cob")));
- LanguageDetection detection = new LanguageDetection(settings.asConfig(), languages);
+ LanguageDetection detection = new LanguageDetection(settings.asConfig(), languages, new HashMap<>());
assertThat(detectLanguageKey(detection, "Foo.java")).isEqualTo("java");
assertThat(detectLanguageKey(detection, "src/Foo.java")).isEqualTo("java");
new MockLanguage("docker", new String[0], new String[] {"*.dockerfile", "*.Dockerfile", "Dockerfile", "Dockerfile.*"}),
new MockLanguage("terraform", new String[] {"tf"}, new String[] {".tf"}),
new MockLanguage("java", new String[0], new String[] {"**/*Test.java"})));
- LanguageDetection detection = new LanguageDetection(settings.asConfig(), languages);
+ LanguageDetection detection = new LanguageDetection(settings.asConfig(), languages, new HashMap<>());
assertThat(detectLanguageKey(detection, fileName)).isEqualTo(expectedLanguageKey);
}
@Test
public void detectLanguageKey_shouldNotFailIfNoLanguage() {
- LanguageDetection detection = spy(new LanguageDetection(settings.asConfig(), new FakeLanguagesRepository(new Languages())));
+ LanguageDetection detection = spy(new LanguageDetection(settings.asConfig(), new FakeLanguagesRepository(new Languages()), new HashMap<>()));
assertThat(detectLanguageKey(detection, "Foo.java")).isNull();
}
public void detectLanguageKey_shouldAllowPluginsToDeclareFileExtensionTwiceForCaseSensitivity() {
LanguagesRepository languages = new FakeLanguagesRepository(new Languages(new MockLanguage("abap", "abap", "ABAP")));
- LanguageDetection detection = new LanguageDetection(settings.asConfig(), languages);
+ LanguageDetection detection = new LanguageDetection(settings.asConfig(), languages, new HashMap<>());
assertThat(detectLanguageKey(detection, "abc.abap")).isEqualTo("abap");
}
@Test
public void detectLanguageKey_shouldFailIfConflictingLanguageSuffix() {
LanguagesRepository languages = new FakeLanguagesRepository(new Languages(new MockLanguage("xml", "xhtml"), new MockLanguage("web", "xhtml")));
- LanguageDetection detection = new LanguageDetection(settings.asConfig(), languages);
+ LanguageDetection detection = new LanguageDetection(settings.asConfig(), languages, new HashMap<>());
assertThatThrownBy(() -> detectLanguageKey(detection, "abc.xhtml"))
.isInstanceOf(MessageException.class)
.hasMessageContaining("Language of file 'abc.xhtml' can not be decided as the file matches patterns of both ")
settings.setProperty("sonar.lang.patterns.xml", "xml/**");
settings.setProperty("sonar.lang.patterns.web", "web/**");
- LanguageDetection detection = new LanguageDetection(settings.asConfig(), languages);
+ LanguageDetection detection = new LanguageDetection(settings.asConfig(), languages, new HashMap<>());
assertThat(detectLanguageKey(detection, "xml/abc.xhtml")).isEqualTo("xml");
assertThat(detectLanguageKey(detection, "web/abc.xhtml")).isEqualTo("web");
}
settings.setProperty("sonar.lang.patterns.abap", "*.abap,*.txt");
settings.setProperty("sonar.lang.patterns.cobol", "*.cobol,*.txt");
- LanguageDetection detection = new LanguageDetection(settings.asConfig(), languages);
+ LanguageDetection detection = new LanguageDetection(settings.asConfig(), languages, new HashMap<>());
assertThat(detectLanguageKey(detection, "abc.abap")).isEqualTo("abap");
assertThat(detectLanguageKey(detection, "abc.cobol")).isEqualTo("cobol");
.hasMessageContaining("sonar.lang.patterns.cobol : *.cobol,*.txt");
}
+ @Test
+ public void should_cache_detected_language_by_file_path() {
+ Map<String, org.sonar.scanner.repository.language.Language> languageCacheSpy = spy(new HashMap<>());
+ LanguagesRepository languages = new FakeLanguagesRepository(new Languages(
+ new MockLanguage("java", "java", "jav"), new MockLanguage("cobol", "cbl", "cob")));
+ LanguageDetection detection = new LanguageDetection(settings.asConfig(), languages, languageCacheSpy);
+
+ assertThat(detectLanguageKey(detection, "Foo.java")).isEqualTo("java");
+ assertThat(detectLanguageKey(detection, "Foo.java")).isEqualTo("java");
+ verify(languageCacheSpy, times(1)).put(endsWith("/Foo.java"), any(org.sonar.scanner.repository.language.Language.class));
+ verify(languageCacheSpy, times(2)).get(endsWith("/Foo.java"));
+ }
+
private String detectLanguageKey(LanguageDetection detection, String path) {
org.sonar.scanner.repository.language.Language language = detection.language(new File(temp.getRoot(), path).toPath(), Paths.get(path));
return language != null ? language.key() : null;
--- /dev/null
+sonar.organization=org1
+sonar.projectKey=sample-with-empty-file
+sonar.projectName=Sample With Empty
+sonar.projectVersion=0.1-SNAPSHOT
+sonar.sources=xources
--- /dev/null
+package hello;
+
+public class HelloJava {
+
+ public static void main(String[] args) {
+ System.out.println("Hello");
+ }
+}
--- /dev/null
+this file should be excluded from indexing.
--- /dev/null
+this file should ALSO be excluded from indexing.
projects / sonarqube.git / commitdiff