testCompile 'org.assertj:assertj-core' | testCompile 'org.assertj:assertj-core' | ||||
testCompile 'org.hamcrest:hamcrest-core' | testCompile 'org.hamcrest:hamcrest-core' | ||||
testCompile 'org.mockito:mockito-core' | testCompile 'org.mockito:mockito-core' | ||||
testCompile 'pmd:pmd:4.3' | |||||
} | } |
/* | |||||
* SonarQube | |||||
* Copyright (C) 2009-2022 SonarSource SA | |||||
* mailto:info AT sonarsource DOT com | |||||
* | |||||
* This program is free software; you can redistribute it and/or | |||||
* modify it under the terms of the GNU Lesser General Public | |||||
* License as published by the Free Software Foundation; either | |||||
* version 3 of the License, or (at your option) any later version. | |||||
* | |||||
* This program is distributed in the hope that it will be useful, | |||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of | |||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |||||
* Lesser General Public License for more details. | |||||
* | |||||
* You should have received a copy of the GNU Lesser General Public License | |||||
* along with this program; if not, write to the Free Software Foundation, | |||||
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. | |||||
*/ | |||||
/** | |||||
* BSD-style license; for more info see http://pmd.sourceforge.net/license.html | |||||
*/ | |||||
package net.sourceforge.pmd.cpd; | |||||
import java.io.File; | |||||
import java.io.FileInputStream; | |||||
import java.io.InputStreamReader; | |||||
import java.io.LineNumberReader; | |||||
import java.io.Reader; | |||||
import java.io.StringReader; | |||||
import java.lang.ref.SoftReference; | |||||
import java.util.ArrayList; | |||||
import java.util.List; | |||||
/** | |||||
* <p>Not intended to be instantiated by clients.</p> | |||||
* | |||||
* @since 2.2 | |||||
* @deprecated since 5.5 | |||||
*/ | |||||
@Deprecated | |||||
public class SourceCode { | |||||
public static final String EOL = System.getProperty("line.separator", "\n"); | |||||
public abstract static class CodeLoader { | |||||
private SoftReference<List<String>> code; | |||||
public List<String> getCode() { | |||||
List<String> c = null; | |||||
if (code != null) { | |||||
c = code.get(); | |||||
} | |||||
if (c != null) { | |||||
return c; | |||||
} | |||||
this.code = new SoftReference<>(load()); | |||||
return code.get(); | |||||
} | |||||
public abstract String getFileName(); | |||||
protected abstract Reader getReader() throws Exception; | |||||
protected List<String> load() { | |||||
try (LineNumberReader lnr = new LineNumberReader(getReader())) { | |||||
List<String> lines = new ArrayList<>(); | |||||
String currentLine; | |||||
while ((currentLine = lnr.readLine()) != null) { | |||||
lines.add(currentLine); | |||||
} | |||||
return lines; | |||||
} catch (Exception e) { | |||||
throw new IllegalStateException("Problem while reading " + getFileName() + ":" + e.getMessage(), e); | |||||
} | |||||
} | |||||
} | |||||
public static class FileCodeLoader extends CodeLoader { | |||||
private File file; | |||||
private String encoding; | |||||
public FileCodeLoader(File file, String encoding) { | |||||
this.file = file; | |||||
this.encoding = encoding; | |||||
} | |||||
@Override | |||||
public Reader getReader() throws Exception { | |||||
return new InputStreamReader(new FileInputStream(file), encoding); | |||||
} | |||||
@Override | |||||
public String getFileName() { | |||||
return this.file.getAbsolutePath(); | |||||
} | |||||
} | |||||
public static class StringCodeLoader extends CodeLoader { | |||||
public static final String DEFAULT_NAME = "CODE_LOADED_FROM_STRING"; | |||||
private String sourceCode; | |||||
private String name; | |||||
public StringCodeLoader(String code) { | |||||
this(code, DEFAULT_NAME); | |||||
} | |||||
public StringCodeLoader(String code, String name) { | |||||
this.sourceCode = code; | |||||
this.name = name; | |||||
} | |||||
@Override | |||||
public Reader getReader() { | |||||
return new StringReader(sourceCode); | |||||
} | |||||
@Override | |||||
public String getFileName() { | |||||
return name; | |||||
} | |||||
} | |||||
private CodeLoader cl; | |||||
public SourceCode(CodeLoader cl) { | |||||
this.cl = cl; | |||||
} | |||||
public List<String> getCode() { | |||||
return cl.getCode(); | |||||
} | |||||
public StringBuffer getCodeBuffer() { | |||||
StringBuffer sb = new StringBuffer(); | |||||
List<String> lines = cl.getCode(); | |||||
for (String line : lines) { | |||||
sb.append(line); | |||||
sb.append(EOL); | |||||
} | |||||
return sb; | |||||
} | |||||
public String getSlice(int startLine, int endLine) { | |||||
StringBuffer sb = new StringBuffer(); | |||||
List lines = cl.getCode(); | |||||
for (int i = (startLine == 0 ? startLine : (startLine - 1)); i < endLine && i < lines.size(); i++) { | |||||
if (sb.length() != 0) { | |||||
sb.append(EOL); | |||||
} | |||||
sb.append((String) lines.get(i)); | |||||
} | |||||
return sb.toString(); | |||||
} | |||||
/** | |||||
* Within Sonar Ecosystem - absolute path to file containing code, | |||||
* whereas in fact existence of such file not guaranteed - see {@link StringCodeLoader}. | |||||
*/ | |||||
public String getFileName() { | |||||
return cl.getFileName(); | |||||
} | |||||
} |
/* | |||||
* SonarQube | |||||
* Copyright (C) 2009-2022 SonarSource SA | |||||
* mailto:info AT sonarsource DOT com | |||||
* | |||||
* This program is free software; you can redistribute it and/or | |||||
* modify it under the terms of the GNU Lesser General Public | |||||
* License as published by the Free Software Foundation; either | |||||
* version 3 of the License, or (at your option) any later version. | |||||
* | |||||
* This program is distributed in the hope that it will be useful, | |||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of | |||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |||||
* Lesser General Public License for more details. | |||||
* | |||||
* You should have received a copy of the GNU Lesser General Public License | |||||
* along with this program; if not, write to the Free Software Foundation, | |||||
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. | |||||
*/ | |||||
/** | |||||
* BSD-style license; for more info see http://pmd.sourceforge.net/license.html | |||||
*/ | |||||
package net.sourceforge.pmd.cpd; | |||||
import java.util.HashMap; | |||||
import java.util.Map; | |||||
/** | |||||
* @since 2.2 | |||||
* @deprecated since 5.5 | |||||
*/ | |||||
@Deprecated | |||||
public class TokenEntry implements Comparable<TokenEntry> { | |||||
private static final Map<String, Integer> TOKENS = new HashMap<>(); | |||||
private static int tokenCount = 0; | |||||
/** | |||||
* Shared instance of end-of-file token. | |||||
* | |||||
* <p>Not intended to be used by clients - {@link #getEOF()} should be used instead.</p> | |||||
*/ | |||||
public static final TokenEntry EOF = new TokenEntry(); | |||||
private String tokenSrcID; | |||||
private int beginLine; | |||||
private int index; | |||||
private int identifier; | |||||
private int hashCode; | |||||
private final String value; | |||||
private TokenEntry() { | |||||
this.identifier = 0; | |||||
this.tokenSrcID = "EOFMarker"; | |||||
this.value = ""; | |||||
} | |||||
/** | |||||
* @param image string representation of token | |||||
* @param tokenSrcID within Sonar Ecosystem - absolute path to file, otherwise current implementation of sonar-cpd-plugin will not work | |||||
* @param beginLine number of line | |||||
*/ | |||||
public TokenEntry(String image, String tokenSrcID, int beginLine) { | |||||
Integer i = TOKENS.get(image); | |||||
if (i == null) { | |||||
i = TOKENS.size() + 1; | |||||
TOKENS.put(image, i); | |||||
} | |||||
this.identifier = i; | |||||
this.tokenSrcID = tokenSrcID; | |||||
this.beginLine = beginLine; | |||||
this.index = tokenCount++; | |||||
this.value = image; | |||||
} | |||||
/** | |||||
* For internal use only. | |||||
* | |||||
* @since 2.14 | |||||
*/ | |||||
public String getValue() { | |||||
return value; | |||||
} | |||||
/** | |||||
* End-of-file token. | |||||
*/ | |||||
public static TokenEntry getEOF() { | |||||
tokenCount++; | |||||
return EOF; | |||||
} | |||||
public static void clearImages() { | |||||
TOKENS.clear(); | |||||
tokenCount = 0; | |||||
} | |||||
public String getTokenSrcID() { | |||||
return tokenSrcID; | |||||
} | |||||
public int getBeginLine() { | |||||
return beginLine; | |||||
} | |||||
public int getIdentifier() { | |||||
return this.identifier; | |||||
} | |||||
public int getIndex() { | |||||
return this.index; | |||||
} | |||||
@Override | |||||
public int hashCode() { | |||||
return hashCode; | |||||
} | |||||
public void setHashCode(int hashCode) { | |||||
this.hashCode = hashCode; | |||||
} | |||||
@Override | |||||
public boolean equals(Object o) { | |||||
if (!(o instanceof TokenEntry)) { | |||||
return false; | |||||
} | |||||
TokenEntry other = (TokenEntry) o; | |||||
return other.hashCode == hashCode; | |||||
} | |||||
@Override | |||||
public int compareTo(TokenEntry other) { | |||||
return getIndex() - other.getIndex(); | |||||
} | |||||
@Override | |||||
public String toString() { | |||||
StringBuilder sb = new StringBuilder("TokenEntry{"); | |||||
sb.append("tokenSrcID='").append(tokenSrcID).append('\''); | |||||
sb.append(", beginLine=").append(beginLine); | |||||
sb.append(", index=").append(index); | |||||
sb.append(", identifier=").append(identifier); | |||||
sb.append(", hashCode=").append(hashCode); | |||||
sb.append(", value='").append(value).append('\''); | |||||
sb.append('}'); | |||||
return sb.toString(); | |||||
} | |||||
} |
/* | |||||
* SonarQube | |||||
* Copyright (C) 2009-2022 SonarSource SA | |||||
* mailto:info AT sonarsource DOT com | |||||
* | |||||
* This program is free software; you can redistribute it and/or | |||||
* modify it under the terms of the GNU Lesser General Public | |||||
* License as published by the Free Software Foundation; either | |||||
* version 3 of the License, or (at your option) any later version. | |||||
* | |||||
* This program is distributed in the hope that it will be useful, | |||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of | |||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |||||
* Lesser General Public License for more details. | |||||
* | |||||
* You should have received a copy of the GNU Lesser General Public License | |||||
* along with this program; if not, write to the Free Software Foundation, | |||||
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. | |||||
*/ | |||||
/** | |||||
* BSD-style license; for more info see http://pmd.sourceforge.net/license.html | |||||
*/ | |||||
package net.sourceforge.pmd.cpd; | |||||
import java.io.IOException; | |||||
/** | |||||
* A tokenizer is responsible to return a token list for the provided input file (see {@link SourceCode#getFileName()}. | |||||
* Tokens are basically list of non empty words in a file but you can also do some "anonymization" to ignore litteral differences. | |||||
* | |||||
* For example if you have a first file: | |||||
* <pre> | |||||
* public class MyClass1 { | |||||
* int foo1; | |||||
* } | |||||
* </pre> | |||||
* and a second file: | |||||
* <pre> | |||||
* public class MyClass2 { | |||||
* int foo2; | |||||
* } | |||||
* </pre> | |||||
* Then in both cases your tokenizer could return the following (line, image) list: | |||||
* <pre>(1,public),(1,class),(1,LITERAL),(1,{),(2,int),(2,LITERAL),(2,;),(3,})</pre> | |||||
* in this case the two files will be considered as duplicate. | |||||
* | |||||
* @since 2.2 | |||||
* @deprecated since 5.5 | |||||
*/ | |||||
@Deprecated | |||||
public interface Tokenizer { | |||||
void tokenize(SourceCode sourceFile, Tokens tokenEntries) throws IOException; | |||||
} |
/* | |||||
* SonarQube | |||||
* Copyright (C) 2009-2022 SonarSource SA | |||||
* mailto:info AT sonarsource DOT com | |||||
* | |||||
* This program is free software; you can redistribute it and/or | |||||
* modify it under the terms of the GNU Lesser General Public | |||||
* License as published by the Free Software Foundation; either | |||||
* version 3 of the License, or (at your option) any later version. | |||||
* | |||||
* This program is distributed in the hope that it will be useful, | |||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of | |||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |||||
* Lesser General Public License for more details. | |||||
* | |||||
* You should have received a copy of the GNU Lesser General Public License | |||||
* along with this program; if not, write to the Free Software Foundation, | |||||
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. | |||||
*/ | |||||
/** | |||||
* BSD-style license; for more info see http://pmd.sourceforge.net/license.html | |||||
*/ | |||||
package net.sourceforge.pmd.cpd; | |||||
import java.util.ArrayList; | |||||
import java.util.Iterator; | |||||
import java.util.List; | |||||
/** | |||||
* <p>Not intended to be instantiated by clients.</p> | |||||
* | |||||
* @since 2.2 | |||||
* @deprecated since 5.5 | |||||
*/ | |||||
@Deprecated | |||||
public class Tokens { | |||||
private List<TokenEntry> entries = new ArrayList<>(); | |||||
public void add(TokenEntry tokenEntry) { | |||||
this.entries.add(tokenEntry); | |||||
} | |||||
public Iterator<TokenEntry> iterator() { | |||||
return entries.iterator(); | |||||
} | |||||
public int size() { | |||||
return entries.size(); | |||||
} | |||||
public List<TokenEntry> getTokens() { | |||||
return entries; | |||||
} | |||||
} |
/* | |||||
* SonarQube | |||||
* Copyright (C) 2009-2022 SonarSource SA | |||||
* mailto:info AT sonarsource DOT com | |||||
* | |||||
* This program is free software; you can redistribute it and/or | |||||
* modify it under the terms of the GNU Lesser General Public | |||||
* License as published by the Free Software Foundation; either | |||||
* version 3 of the License, or (at your option) any later version. | |||||
* | |||||
* This program is distributed in the hope that it will be useful, | |||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of | |||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |||||
* Lesser General Public License for more details. | |||||
* | |||||
* You should have received a copy of the GNU Lesser General Public License | |||||
* along with this program; if not, write to the Free Software Foundation, | |||||
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. | |||||
*/ | |||||
/** | |||||
* Provides a basic framework to sequentially read any kind of character stream and create list of tokens. | |||||
* | |||||
* The entry point of this framework is the {@link org.sonar.duplications.token.TokenChunker} class. | |||||
*/ | |||||
@ParametersAreNonnullByDefault | |||||
package net.sourceforge.pmd.cpd; | |||||
import javax.annotation.ParametersAreNonnullByDefault; | |||||
/* | |||||
* SonarQube | |||||
* Copyright (C) 2009-2022 SonarSource SA | |||||
* mailto:info AT sonarsource DOT com | |||||
* | |||||
* This program is free software; you can redistribute it and/or | |||||
* modify it under the terms of the GNU Lesser General Public | |||||
* License as published by the Free Software Foundation; either | |||||
* version 3 of the License, or (at your option) any later version. | |||||
* | |||||
* This program is distributed in the hope that it will be useful, | |||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of | |||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |||||
* Lesser General Public License for more details. | |||||
* | |||||
* You should have received a copy of the GNU Lesser General Public License | |||||
* along with this program; if not, write to the Free Software Foundation, | |||||
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. | |||||
*/ | |||||
package org.sonar.duplications.cpd; | |||||
import java.util.List; | |||||
import net.sourceforge.pmd.cpd.SourceCode.CodeLoader; | |||||
public abstract class CodeLoaderWithoutCache extends CodeLoader { | |||||
@Override | |||||
public final List<String> getCode() { | |||||
return load(); | |||||
} | |||||
} |
/* | |||||
* SonarQube | |||||
* Copyright (C) 2009-2022 SonarSource SA | |||||
* mailto:info AT sonarsource DOT com | |||||
* | |||||
* This program is free software; you can redistribute it and/or | |||||
* modify it under the terms of the GNU Lesser General Public | |||||
* License as published by the Free Software Foundation; either | |||||
* version 3 of the License, or (at your option) any later version. | |||||
* | |||||
* This program is distributed in the hope that it will be useful, | |||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of | |||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |||||
* Lesser General Public License for more details. | |||||
* | |||||
* You should have received a copy of the GNU Lesser General Public License | |||||
* along with this program; if not, write to the Free Software Foundation, | |||||
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. | |||||
*/ | |||||
package org.sonar.duplications.cpd; | |||||
import java.io.Reader; | |||||
public class FileCodeLoaderWithoutCache extends CodeLoaderWithoutCache { | |||||
private final String fileName; | |||||
private final Reader fileReader; | |||||
public FileCodeLoaderWithoutCache(String fileName, Reader fileReader) { | |||||
this.fileName = fileName; | |||||
this.fileReader = fileReader; | |||||
} | |||||
@Override | |||||
public Reader getReader() throws Exception { | |||||
return fileReader; | |||||
} | |||||
@Override | |||||
public String getFileName() { | |||||
return fileName; | |||||
} | |||||
} |
/* | |||||
* SonarQube | |||||
* Copyright (C) 2009-2022 SonarSource SA | |||||
* mailto:info AT sonarsource DOT com | |||||
* | |||||
* This program is free software; you can redistribute it and/or | |||||
* modify it under the terms of the GNU Lesser General Public | |||||
* License as published by the Free Software Foundation; either | |||||
* version 3 of the License, or (at your option) any later version. | |||||
* | |||||
* This program is distributed in the hope that it will be useful, | |||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of | |||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |||||
* Lesser General Public License for more details. | |||||
* | |||||
* You should have received a copy of the GNU Lesser General Public License | |||||
* along with this program; if not, write to the Free Software Foundation, | |||||
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. | |||||
*/ | |||||
@ParametersAreNonnullByDefault | |||||
package org.sonar.duplications.cpd; | |||||
import javax.annotation.ParametersAreNonnullByDefault; | |||||
/* | |||||
* SonarQube | |||||
* Copyright (C) 2009-2022 SonarSource SA | |||||
* mailto:info AT sonarsource DOT com | |||||
* | |||||
* This program is free software; you can redistribute it and/or | |||||
* modify it under the terms of the GNU Lesser General Public | |||||
* License as published by the Free Software Foundation; either | |||||
* version 3 of the License, or (at your option) any later version. | |||||
* | |||||
* This program is distributed in the hope that it will be useful, | |||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of | |||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |||||
* Lesser General Public License for more details. | |||||
* | |||||
* You should have received a copy of the GNU Lesser General Public License | |||||
* along with this program; if not, write to the Free Software Foundation, | |||||
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. | |||||
*/ | |||||
package org.sonar.duplications.internal.pmd; | |||||
import java.io.Reader; | |||||
import java.util.ArrayList; | |||||
import java.util.List; | |||||
import net.sourceforge.pmd.cpd.SourceCode; | |||||
import net.sourceforge.pmd.cpd.TokenEntry; | |||||
import net.sourceforge.pmd.cpd.Tokenizer; | |||||
import net.sourceforge.pmd.cpd.Tokens; | |||||
import org.sonar.api.batch.sensor.cpd.internal.TokensLine; | |||||
import org.sonar.duplications.block.Block; | |||||
import org.sonar.duplications.cpd.FileCodeLoaderWithoutCache; | |||||
/** | |||||
* Bridge, which allows to convert list of {@link TokenEntry} produced by {@link Tokenizer} into list of {@link TokensLine}s. | |||||
*/ | |||||
public class TokenizerBridge { | |||||
private final Tokenizer tokenizer; | |||||
private final PmdBlockChunker blockBuilder; | |||||
public TokenizerBridge(Tokenizer tokenizer, int blockSize) { | |||||
this.tokenizer = tokenizer; | |||||
this.blockBuilder = new PmdBlockChunker(blockSize); | |||||
} | |||||
public List<Block> chunk(String resourceId, String fileName, Reader fileReader) { | |||||
return blockBuilder.chunk(resourceId, chunk(fileName, fileReader)); | |||||
} | |||||
public List<TokensLine> chunk(String fileName, Reader fileReader) { | |||||
SourceCode sourceCode = new SourceCode(new FileCodeLoaderWithoutCache(fileName, fileReader)); | |||||
Tokens tokens = new Tokens(); | |||||
TokenEntry.clearImages(); | |||||
try { | |||||
tokenizer.tokenize(sourceCode, tokens); | |||||
} catch (RuntimeException e) { | |||||
throw e; | |||||
} catch (Exception e) { | |||||
throw new RuntimeException(e); | |||||
} | |||||
TokenEntry.clearImages(); | |||||
return convert(tokens.getTokens()); | |||||
} | |||||
/** | |||||
* We expect that implementation of {@link Tokenizer} is correct: | |||||
* tokens ordered by occurrence in source code and last token is EOF. | |||||
*/ | |||||
public static List<TokensLine> convert(List<TokenEntry> tokens) { | |||||
List<TokensLine> result = new ArrayList<>(); | |||||
StringBuilder sb = new StringBuilder(); | |||||
int startLine = Integer.MIN_VALUE; | |||||
int startIndex = 0; | |||||
int currentIndex = 0; | |||||
for (TokenEntry token : tokens) { | |||||
if (token != TokenEntry.EOF) { | |||||
String value = token.getValue(); | |||||
int line = token.getBeginLine(); | |||||
if (line != startLine) { | |||||
addNewTokensLine(result, startIndex, currentIndex, startLine, sb); | |||||
startIndex = currentIndex + 1; | |||||
startLine = line; | |||||
} | |||||
currentIndex++; | |||||
sb.append(value); | |||||
} | |||||
} | |||||
addNewTokensLine(result, startIndex, currentIndex, startLine, sb); | |||||
return result; | |||||
} | |||||
private static void addNewTokensLine(List<TokensLine> result, int startUnit, int endUnit, int startLine, StringBuilder sb) { | |||||
if (sb.length() != 0) { | |||||
result.add(new TokensLine(startUnit, endUnit, startLine, sb.toString())); | |||||
sb.setLength(0); | |||||
} | |||||
} | |||||
} |
/* | |||||
* SonarQube | |||||
* Copyright (C) 2009-2022 SonarSource SA | |||||
* mailto:info AT sonarsource DOT com | |||||
* | |||||
* This program is free software; you can redistribute it and/or | |||||
* modify it under the terms of the GNU Lesser General Public | |||||
* License as published by the Free Software Foundation; either | |||||
* version 3 of the License, or (at your option) any later version. | |||||
* | |||||
* This program is distributed in the hope that it will be useful, | |||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of | |||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |||||
* Lesser General Public License for more details. | |||||
* | |||||
* You should have received a copy of the GNU Lesser General Public License | |||||
* along with this program; if not, write to the Free Software Foundation, | |||||
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. | |||||
*/ | |||||
package net.sourceforge.pmd.cpd; | |||||
import org.junit.Before; | |||||
import org.junit.Test; | |||||
import static org.hamcrest.CoreMatchers.equalTo; | |||||
import static org.hamcrest.CoreMatchers.sameInstance; | |||||
import static org.junit.Assert.assertThat; | |||||
public class TokenEntryTest { | |||||
@Before | |||||
public void setUp() { | |||||
TokenEntry.clearImages(); | |||||
} | |||||
@Test | |||||
public void testNewTokenEntry() { | |||||
TokenEntry entry = new TokenEntry("token1", "src1", 1); | |||||
assertThat(entry.getValue(), equalTo("token1")); | |||||
assertThat(entry.getBeginLine(), equalTo(1)); | |||||
entry = new TokenEntry("token2", "src2", 2); | |||||
assertThat(entry.getValue(), equalTo("token2")); | |||||
assertThat(entry.getBeginLine(), equalTo(2)); | |||||
} | |||||
@Test | |||||
public void testGetEOF() { | |||||
assertThat(TokenEntry.getEOF(), sameInstance(TokenEntry.getEOF())); | |||||
} | |||||
} |
/* | |||||
* SonarQube | |||||
* Copyright (C) 2009-2022 SonarSource SA | |||||
* mailto:info AT sonarsource DOT com | |||||
* | |||||
* This program is free software; you can redistribute it and/or | |||||
* modify it under the terms of the GNU Lesser General Public | |||||
* License as published by the Free Software Foundation; either | |||||
* version 3 of the License, or (at your option) any later version. | |||||
* | |||||
* This program is distributed in the hope that it will be useful, | |||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of | |||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |||||
* Lesser General Public License for more details. | |||||
* | |||||
* You should have received a copy of the GNU Lesser General Public License | |||||
* along with this program; if not, write to the Free Software Foundation, | |||||
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. | |||||
*/ | |||||
package org.sonar.duplications.internal.pmd; | |||||
import java.io.File; | |||||
import java.io.IOException; | |||||
import java.nio.charset.StandardCharsets; | |||||
import java.nio.file.Files; | |||||
import java.util.Collection; | |||||
import java.util.List; | |||||
import net.sourceforge.pmd.cpd.JavaTokenizer; | |||||
import org.junit.Before; | |||||
import org.junit.Test; | |||||
import org.sonar.duplications.block.Block; | |||||
import org.sonar.duplications.detector.suffixtree.SuffixTreeCloneDetectionAlgorithm; | |||||
import org.sonar.duplications.index.CloneGroup; | |||||
import org.sonar.duplications.index.CloneIndex; | |||||
import org.sonar.duplications.index.ClonePart; | |||||
import org.sonar.duplications.index.PackedMemoryCloneIndex; | |||||
import static org.assertj.core.api.Assertions.assertThat; | |||||
public class PmdBridgeTest { | |||||
private CloneIndex index; | |||||
private TokenizerBridge bridge; | |||||
@Before | |||||
public void setUp() { | |||||
index = new PackedMemoryCloneIndex(); | |||||
bridge = new TokenizerBridge(new JavaTokenizer(), 10); | |||||
} | |||||
@Test | |||||
public void testDuplicationInSingleFile() throws IOException { | |||||
File file = new File("test-resources/org/sonar/duplications/cpd/CPDTest/CPDFile3.java"); | |||||
addToIndex(file); | |||||
List<CloneGroup> duplications = detect(file); | |||||
assertThat(duplications.size()).isOne(); | |||||
CloneGroup duplication = duplications.get(0); | |||||
assertThat(duplication.getOriginPart().getResourceId()).isEqualTo(file.getAbsolutePath()); | |||||
assertThat(duplication.getCloneParts()).hasSize(2); | |||||
assertThat(duplication.getLengthInUnits()).as("length in tokens").isEqualTo(157); | |||||
ClonePart part = duplication.getCloneParts().get(0); | |||||
assertThat(part.getResourceId()).isEqualTo(file.getAbsolutePath()); | |||||
assertThat(part.getStartLine()).isEqualTo(30); | |||||
assertThat(part.getEndLine()).isEqualTo(44); | |||||
} | |||||
@Test | |||||
public void testDuplicationBetweenTwoFiles() throws IOException { | |||||
File file1 = new File("test-resources/org/sonar/duplications/cpd/CPDTest/CPDFile1.java"); | |||||
File file2 = new File("test-resources/org/sonar/duplications/cpd/CPDTest/CPDFile2.java"); | |||||
addToIndex(file1); | |||||
addToIndex(file2); | |||||
List<CloneGroup> duplications = detect(file1); | |||||
assertThat(duplications.size()).isOne(); | |||||
CloneGroup duplication = duplications.get(0); | |||||
assertThat(duplication.getOriginPart().getResourceId()).isEqualTo(file1.getAbsolutePath()); | |||||
ClonePart part1 = new ClonePart(file1.getAbsolutePath(), 1, 18, 41); | |||||
ClonePart part2 = new ClonePart(file2.getAbsolutePath(), 1, 18, 41); | |||||
assertThat(duplication.getCloneParts()).containsOnly(part1, part2); | |||||
assertThat(duplication.getLengthInUnits()).as("length in tokens").isEqualTo(115); | |||||
} | |||||
private List<CloneGroup> detect(File file) { | |||||
Collection<Block> fileBlocks = index.getByResourceId(file.getAbsolutePath()); | |||||
return SuffixTreeCloneDetectionAlgorithm.detect(index, fileBlocks); | |||||
} | |||||
private void addToIndex(File file) throws IOException { | |||||
List<Block> blocks = bridge.chunk(file.getAbsolutePath(), file.getAbsolutePath(), Files.newBufferedReader(file.toPath(), StandardCharsets.UTF_8)); | |||||
for (Block block : blocks) { | |||||
index.insert(block); | |||||
} | |||||
} | |||||
} |
/* | |||||
* SonarQube | |||||
* Copyright (C) 2009-2022 SonarSource SA | |||||
* mailto:info AT sonarsource DOT com | |||||
* | |||||
* This program is free software; you can redistribute it and/or | |||||
* modify it under the terms of the GNU Lesser General Public | |||||
* License as published by the Free Software Foundation; either | |||||
* version 3 of the License, or (at your option) any later version. | |||||
* | |||||
* This program is distributed in the hope that it will be useful, | |||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of | |||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |||||
* Lesser General Public License for more details. | |||||
* | |||||
* You should have received a copy of the GNU Lesser General Public License | |||||
* along with this program; if not, write to the Free Software Foundation, | |||||
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. | |||||
*/ | |||||
package org.sonar.duplications.internal.pmd; | |||||
import java.io.ByteArrayInputStream; | |||||
import java.io.InputStreamReader; | |||||
import java.nio.charset.StandardCharsets; | |||||
import java.util.List; | |||||
import net.sourceforge.pmd.cpd.SourceCode; | |||||
import net.sourceforge.pmd.cpd.TokenEntry; | |||||
import net.sourceforge.pmd.cpd.Tokenizer; | |||||
import net.sourceforge.pmd.cpd.Tokens; | |||||
import org.junit.Before; | |||||
import org.junit.Test; | |||||
import org.sonar.api.batch.sensor.cpd.internal.TokensLine; | |||||
import static org.hamcrest.CoreMatchers.is; | |||||
import static org.junit.Assert.assertThat; | |||||
public class TokenizerBridgeTest { | |||||
private TokenizerBridge bridge; | |||||
@Before | |||||
public void setUp() { | |||||
Tokenizer tokenizer = new Tokenizer() { | |||||
public void tokenize(SourceCode tokens, Tokens tokenEntries) { | |||||
tokenEntries.add(new TokenEntry("t1", "src", 1)); | |||||
tokenEntries.add(new TokenEntry("t2", "src", 1)); | |||||
tokenEntries.add(new TokenEntry("t3", "src", 2)); | |||||
tokenEntries.add(new TokenEntry("t1", "src", 4)); | |||||
tokenEntries.add(new TokenEntry("t3", "src", 4)); | |||||
tokenEntries.add(new TokenEntry("t3", "src", 4)); | |||||
tokenEntries.add(TokenEntry.getEOF()); | |||||
} | |||||
}; | |||||
bridge = new TokenizerBridge(tokenizer, 10); | |||||
} | |||||
@Test | |||||
public void shouldClearCacheInTokenEntry() { | |||||
bridge.chunk("file.txt", new InputStreamReader(new ByteArrayInputStream(new byte[0]), StandardCharsets.UTF_8)); | |||||
TokenEntry token = new TokenEntry("image", "srcId", 0); | |||||
assertThat(token.getIndex(), is(0)); | |||||
assertThat(token.getIdentifier(), is(1)); | |||||
} | |||||
@Test | |||||
public void test() { | |||||
// To be sure that token index will be relative to file - run twice: | |||||
bridge.chunk("file.txt", new InputStreamReader(new ByteArrayInputStream(new byte[0]), StandardCharsets.UTF_8)); | |||||
List<TokensLine> lines = bridge.chunk("file.txt", new InputStreamReader(new ByteArrayInputStream(new byte[0]), StandardCharsets.UTF_8)); | |||||
assertThat(lines.size(), is(3)); | |||||
TokensLine line = lines.get(0); | |||||
// 2 tokens on 1 line | |||||
assertThat(line.getStartUnit(), is(1)); | |||||
assertThat(line.getEndUnit(), is(2)); | |||||
assertThat(line.getStartLine(), is(1)); | |||||
assertThat(line.getEndLine(), is(1)); | |||||
assertThat(line.getHashCode(), is("t1t2".hashCode())); | |||||
line = lines.get(1); | |||||
// 1 token on 2 line | |||||
assertThat(line.getStartUnit(), is(3)); | |||||
assertThat(line.getEndUnit(), is(3)); | |||||
assertThat(line.getStartLine(), is(2)); | |||||
assertThat(line.getEndLine(), is(2)); | |||||
assertThat(line.getHashCode(), is("t3".hashCode())); | |||||
line = lines.get(2); | |||||
// 3 tokens on 4 line | |||||
assertThat(line.getStartUnit(), is(4)); | |||||
assertThat(line.getEndUnit(), is(6)); | |||||
assertThat(line.getStartLine(), is(4)); | |||||
assertThat(line.getEndLine(), is(4)); | |||||
assertThat(line.getHashCode(), is("t1t3t3".hashCode())); | |||||
} | |||||
} |