1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
|
/*
* SonarQube
* Copyright (C) 2009-2025 SonarSource SA
* mailto:info AT sonarsource DOT com
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3 of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program; if not, write to the Free Software Foundation,
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
package org.sonar.xoo.lang;
import java.io.IOException;
import org.apache.commons.io.FileUtils;
import org.sonar.api.batch.fs.FilePredicates;
import org.sonar.api.batch.fs.InputFile;
import org.sonar.api.batch.fs.InputFile.Type;
import org.sonar.api.batch.sensor.Sensor;
import org.sonar.api.batch.sensor.SensorContext;
import org.sonar.api.batch.sensor.SensorDescriptor;
import org.sonar.api.batch.sensor.cpd.NewCpdTokens;
import org.sonar.xoo.Xoo;
/**
* Tokenize files for CPD
*/
public class CpdTokenizerSensor implements Sensor {
private void tokenize(InputFile inputFile, SensorContext context) {
int lineIdx = 1;
NewCpdTokens newCpdTokens = context.newCpdTokens().onFile(inputFile);
try {
StringBuilder sb = new StringBuilder();
for (String line : FileUtils.readLines(inputFile.file(), inputFile.charset())) {
int startOffset = 0;
int endOffset = 0;
for (int i = 0; i < line.length(); i++) {
char c = line.charAt(i);
if (Character.isWhitespace(c)) {
if (sb.length() > 0) {
newCpdTokens.addToken(inputFile.newRange(lineIdx, startOffset, lineIdx, endOffset), sb.toString());
sb.setLength(0);
}
startOffset = endOffset;
} else {
sb.append(c);
}
endOffset++;
}
if (sb.length() > 0) {
newCpdTokens.addToken(inputFile.newRange(lineIdx, startOffset, lineIdx, endOffset), sb.toString());
sb.setLength(0);
}
lineIdx++;
}
} catch (IOException e) {
throw new IllegalStateException("Unable to tokenize", e);
}
newCpdTokens.save();
}
@Override
public void describe(SensorDescriptor descriptor) {
descriptor
.name("Xoo Cpd Tokenizer Sensor")
.onlyOnLanguages(Xoo.KEY);
}
@Override
public void execute(SensorContext context) {
FilePredicates p = context.fileSystem().predicates();
for (InputFile file : context.fileSystem().inputFiles(p.and(p.hasLanguages(Xoo.KEY), p.hasType(Type.MAIN)))) {
tokenize(file, context);
}
}
}
|
