aboutsummaryrefslogtreecommitdiffstats
path: root/sonar-duplications/src/main/java
diff options
context:
space:
mode:
authorPierre <pierre.guillot@sonarsource.com>2022-06-30 11:55:04 +0200
committersonartech <sonartech@sonarsource.com>2022-06-30 20:03:10 +0000
commitba2118d645faad8ef419cc5b301bcaf394d2571e (patch)
treed2ab4bf23ee077c4158e81f020dee262a1bcbe3b /sonar-duplications/src/main/java
parent0e9d44d4b0b4d4017c360bf686dcb1a8855c3f23 (diff)
downloadsonarqube-ba2118d645faad8ef419cc5b301bcaf394d2571e.tar.gz
sonarqube-ba2118d645faad8ef419cc5b301bcaf394d2571e.zip
SONAR-16591 remove PMD dependency and related, deprecated code
Diffstat (limited to 'sonar-duplications/src/main/java')
-rw-r--r--sonar-duplications/src/main/java/net/sourceforge/pmd/cpd/SourceCode.java165
-rw-r--r--sonar-duplications/src/main/java/net/sourceforge/pmd/cpd/TokenEntry.java150
-rw-r--r--sonar-duplications/src/main/java/net/sourceforge/pmd/cpd/Tokenizer.java55
-rw-r--r--sonar-duplications/src/main/java/net/sourceforge/pmd/cpd/Tokens.java56
-rw-r--r--sonar-duplications/src/main/java/net/sourceforge/pmd/cpd/package-info.java29
-rw-r--r--sonar-duplications/src/main/java/org/sonar/duplications/cpd/CodeLoaderWithoutCache.java32
-rw-r--r--sonar-duplications/src/main/java/org/sonar/duplications/cpd/FileCodeLoaderWithoutCache.java43
-rw-r--r--sonar-duplications/src/main/java/org/sonar/duplications/cpd/package-info.java24
-rw-r--r--sonar-duplications/src/main/java/org/sonar/duplications/internal/pmd/TokenizerBridge.java99
9 files changed, 0 insertions, 653 deletions
diff --git a/sonar-duplications/src/main/java/net/sourceforge/pmd/cpd/SourceCode.java b/sonar-duplications/src/main/java/net/sourceforge/pmd/cpd/SourceCode.java
deleted file mode 100644
index ba7fc84021c..00000000000
--- a/sonar-duplications/src/main/java/net/sourceforge/pmd/cpd/SourceCode.java
+++ /dev/null
@@ -1,165 +0,0 @@
-/*
- * SonarQube
- * Copyright (C) 2009-2022 SonarSource SA
- * mailto:info AT sonarsource DOT com
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 3 of the License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public License
- * along with this program; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
- */
-/**
- * BSD-style license; for more info see http://pmd.sourceforge.net/license.html
- */
-package net.sourceforge.pmd.cpd;
-
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.InputStreamReader;
-import java.io.LineNumberReader;
-import java.io.Reader;
-import java.io.StringReader;
-import java.lang.ref.SoftReference;
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * <p>Not intended to be instantiated by clients.</p>
- *
- * @since 2.2
- * @deprecated since 5.5
- */
-@Deprecated
-public class SourceCode {
-
- public static final String EOL = System.getProperty("line.separator", "\n");
-
- public abstract static class CodeLoader {
- private SoftReference<List<String>> code;
-
- public List<String> getCode() {
- List<String> c = null;
- if (code != null) {
- c = code.get();
- }
- if (c != null) {
- return c;
- }
- this.code = new SoftReference<>(load());
- return code.get();
- }
-
- public abstract String getFileName();
-
- protected abstract Reader getReader() throws Exception;
-
- protected List<String> load() {
- try (LineNumberReader lnr = new LineNumberReader(getReader())) {
- List<String> lines = new ArrayList<>();
- String currentLine;
- while ((currentLine = lnr.readLine()) != null) {
- lines.add(currentLine);
- }
- return lines;
- } catch (Exception e) {
- throw new IllegalStateException("Problem while reading " + getFileName() + ":" + e.getMessage(), e);
- }
- }
- }
-
- public static class FileCodeLoader extends CodeLoader {
- private File file;
- private String encoding;
-
- public FileCodeLoader(File file, String encoding) {
- this.file = file;
- this.encoding = encoding;
- }
-
- @Override
- public Reader getReader() throws Exception {
- return new InputStreamReader(new FileInputStream(file), encoding);
- }
-
- @Override
- public String getFileName() {
- return this.file.getAbsolutePath();
- }
- }
-
- public static class StringCodeLoader extends CodeLoader {
- public static final String DEFAULT_NAME = "CODE_LOADED_FROM_STRING";
-
- private String sourceCode;
-
- private String name;
-
- public StringCodeLoader(String code) {
- this(code, DEFAULT_NAME);
- }
-
- public StringCodeLoader(String code, String name) {
- this.sourceCode = code;
- this.name = name;
- }
-
- @Override
- public Reader getReader() {
- return new StringReader(sourceCode);
- }
-
- @Override
- public String getFileName() {
- return name;
- }
- }
-
- private CodeLoader cl;
-
- public SourceCode(CodeLoader cl) {
- this.cl = cl;
- }
-
- public List<String> getCode() {
- return cl.getCode();
- }
-
- public StringBuffer getCodeBuffer() {
- StringBuffer sb = new StringBuffer();
- List<String> lines = cl.getCode();
- for (String line : lines) {
- sb.append(line);
- sb.append(EOL);
- }
- return sb;
- }
-
- public String getSlice(int startLine, int endLine) {
- StringBuffer sb = new StringBuffer();
- List lines = cl.getCode();
- for (int i = (startLine == 0 ? startLine : (startLine - 1)); i < endLine && i < lines.size(); i++) {
- if (sb.length() != 0) {
- sb.append(EOL);
- }
- sb.append((String) lines.get(i));
- }
- return sb.toString();
- }
-
- /**
- * Within Sonar Ecosystem - absolute path to file containing code,
- * whereas in fact existence of such file not guaranteed - see {@link StringCodeLoader}.
- */
- public String getFileName() {
- return cl.getFileName();
- }
-}
diff --git a/sonar-duplications/src/main/java/net/sourceforge/pmd/cpd/TokenEntry.java b/sonar-duplications/src/main/java/net/sourceforge/pmd/cpd/TokenEntry.java
deleted file mode 100644
index 7bf20176431..00000000000
--- a/sonar-duplications/src/main/java/net/sourceforge/pmd/cpd/TokenEntry.java
+++ /dev/null
@@ -1,150 +0,0 @@
-/*
- * SonarQube
- * Copyright (C) 2009-2022 SonarSource SA
- * mailto:info AT sonarsource DOT com
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 3 of the License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public License
- * along with this program; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
- */
-/**
- * BSD-style license; for more info see http://pmd.sourceforge.net/license.html
- */
-package net.sourceforge.pmd.cpd;
-
-import java.util.HashMap;
-import java.util.Map;
-
-/**
- * @since 2.2
- * @deprecated since 5.5
- */
-@Deprecated
-public class TokenEntry implements Comparable<TokenEntry> {
-
- private static final Map<String, Integer> TOKENS = new HashMap<>();
- private static int tokenCount = 0;
-
- /**
- * Shared instance of end-of-file token.
- *
- * <p>Not intended to be used by clients - {@link #getEOF()} should be used instead.</p>
- */
- public static final TokenEntry EOF = new TokenEntry();
-
- private String tokenSrcID;
- private int beginLine;
- private int index;
- private int identifier;
- private int hashCode;
-
- private final String value;
-
- private TokenEntry() {
- this.identifier = 0;
- this.tokenSrcID = "EOFMarker";
- this.value = "";
- }
-
- /**
- * @param image string representation of token
- * @param tokenSrcID within Sonar Ecosystem - absolute path to file, otherwise current implementation of sonar-cpd-plugin will not work
- * @param beginLine number of line
- */
- public TokenEntry(String image, String tokenSrcID, int beginLine) {
- Integer i = TOKENS.get(image);
- if (i == null) {
- i = TOKENS.size() + 1;
- TOKENS.put(image, i);
- }
- this.identifier = i;
- this.tokenSrcID = tokenSrcID;
- this.beginLine = beginLine;
- this.index = tokenCount++;
- this.value = image;
- }
-
- /**
- * For internal use only.
- *
- * @since 2.14
- */
- public String getValue() {
- return value;
- }
-
- /**
- * End-of-file token.
- */
- public static TokenEntry getEOF() {
- tokenCount++;
- return EOF;
- }
-
- public static void clearImages() {
- TOKENS.clear();
- tokenCount = 0;
- }
-
- public String getTokenSrcID() {
- return tokenSrcID;
- }
-
- public int getBeginLine() {
- return beginLine;
- }
-
- public int getIdentifier() {
- return this.identifier;
- }
-
- public int getIndex() {
- return this.index;
- }
-
- @Override
- public int hashCode() {
- return hashCode;
- }
-
- public void setHashCode(int hashCode) {
- this.hashCode = hashCode;
- }
-
- @Override
- public boolean equals(Object o) {
- if (!(o instanceof TokenEntry)) {
- return false;
- }
- TokenEntry other = (TokenEntry) o;
- return other.hashCode == hashCode;
- }
-
- @Override
- public int compareTo(TokenEntry other) {
- return getIndex() - other.getIndex();
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("TokenEntry{");
- sb.append("tokenSrcID='").append(tokenSrcID).append('\'');
- sb.append(", beginLine=").append(beginLine);
- sb.append(", index=").append(index);
- sb.append(", identifier=").append(identifier);
- sb.append(", hashCode=").append(hashCode);
- sb.append(", value='").append(value).append('\'');
- sb.append('}');
- return sb.toString();
- }
-}
diff --git a/sonar-duplications/src/main/java/net/sourceforge/pmd/cpd/Tokenizer.java b/sonar-duplications/src/main/java/net/sourceforge/pmd/cpd/Tokenizer.java
deleted file mode 100644
index c5580a905a3..00000000000
--- a/sonar-duplications/src/main/java/net/sourceforge/pmd/cpd/Tokenizer.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * SonarQube
- * Copyright (C) 2009-2022 SonarSource SA
- * mailto:info AT sonarsource DOT com
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 3 of the License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public License
- * along with this program; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
- */
-/**
- * BSD-style license; for more info see http://pmd.sourceforge.net/license.html
- */
-package net.sourceforge.pmd.cpd;
-
-import java.io.IOException;
-
-/**
- * A tokenizer is responsible to return a token list for the provided input file (see {@link SourceCode#getFileName()}.
- * Tokens are basically list of non empty words in a file but you can also do some "anonymization" to ignore litteral differences.
- *
- * For example if you have a first file:
- * <pre>
- * public class MyClass1 {
- * int foo1;
- * }
- * </pre>
- * and a second file:
- * <pre>
- * public class MyClass2 {
- * int foo2;
- * }
- * </pre>
- * Then in both cases your tokenizer could return the following (line, image) list:
- * <pre>(1,public),(1,class),(1,LITERAL),(1,{),(2,int),(2,LITERAL),(2,;),(3,})</pre>
- * in this case the two files will be considered as duplicate.
- *
- * @since 2.2
- * @deprecated since 5.5
- */
-@Deprecated
-public interface Tokenizer {
-
- void tokenize(SourceCode sourceFile, Tokens tokenEntries) throws IOException;
-
-}
diff --git a/sonar-duplications/src/main/java/net/sourceforge/pmd/cpd/Tokens.java b/sonar-duplications/src/main/java/net/sourceforge/pmd/cpd/Tokens.java
deleted file mode 100644
index df9799380f4..00000000000
--- a/sonar-duplications/src/main/java/net/sourceforge/pmd/cpd/Tokens.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * SonarQube
- * Copyright (C) 2009-2022 SonarSource SA
- * mailto:info AT sonarsource DOT com
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 3 of the License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public License
- * along with this program; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
- */
-/**
- * BSD-style license; for more info see http://pmd.sourceforge.net/license.html
- */
-package net.sourceforge.pmd.cpd;
-
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
-
-/**
- * <p>Not intended to be instantiated by clients.</p>
- *
- * @since 2.2
- * @deprecated since 5.5
- */
-@Deprecated
-public class Tokens {
-
- private List<TokenEntry> entries = new ArrayList<>();
-
- public void add(TokenEntry tokenEntry) {
- this.entries.add(tokenEntry);
- }
-
- public Iterator<TokenEntry> iterator() {
- return entries.iterator();
- }
-
- public int size() {
- return entries.size();
- }
-
- public List<TokenEntry> getTokens() {
- return entries;
- }
-
-}
diff --git a/sonar-duplications/src/main/java/net/sourceforge/pmd/cpd/package-info.java b/sonar-duplications/src/main/java/net/sourceforge/pmd/cpd/package-info.java
deleted file mode 100644
index 2b1977c871b..00000000000
--- a/sonar-duplications/src/main/java/net/sourceforge/pmd/cpd/package-info.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * SonarQube
- * Copyright (C) 2009-2022 SonarSource SA
- * mailto:info AT sonarsource DOT com
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 3 of the License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public License
- * along with this program; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
- */
-/**
- * Provides a basic framework to sequentially read any kind of character stream and create list of tokens.
- *
- * The entry point of this framework is the {@link org.sonar.duplications.token.TokenChunker} class.
- */
-@ParametersAreNonnullByDefault
-package net.sourceforge.pmd.cpd;
-
-import javax.annotation.ParametersAreNonnullByDefault;
-
diff --git a/sonar-duplications/src/main/java/org/sonar/duplications/cpd/CodeLoaderWithoutCache.java b/sonar-duplications/src/main/java/org/sonar/duplications/cpd/CodeLoaderWithoutCache.java
deleted file mode 100644
index 41c48da888d..00000000000
--- a/sonar-duplications/src/main/java/org/sonar/duplications/cpd/CodeLoaderWithoutCache.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * SonarQube
- * Copyright (C) 2009-2022 SonarSource SA
- * mailto:info AT sonarsource DOT com
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 3 of the License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public License
- * along with this program; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
- */
-package org.sonar.duplications.cpd;
-
-import java.util.List;
-
-import net.sourceforge.pmd.cpd.SourceCode.CodeLoader;
-
-public abstract class CodeLoaderWithoutCache extends CodeLoader {
-
- @Override
- public final List<String> getCode() {
- return load();
- }
-}
diff --git a/sonar-duplications/src/main/java/org/sonar/duplications/cpd/FileCodeLoaderWithoutCache.java b/sonar-duplications/src/main/java/org/sonar/duplications/cpd/FileCodeLoaderWithoutCache.java
deleted file mode 100644
index f30db7f44e7..00000000000
--- a/sonar-duplications/src/main/java/org/sonar/duplications/cpd/FileCodeLoaderWithoutCache.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * SonarQube
- * Copyright (C) 2009-2022 SonarSource SA
- * mailto:info AT sonarsource DOT com
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 3 of the License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public License
- * along with this program; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
- */
-package org.sonar.duplications.cpd;
-
-import java.io.Reader;
-
-public class FileCodeLoaderWithoutCache extends CodeLoaderWithoutCache {
-
- private final String fileName;
- private final Reader fileReader;
-
- public FileCodeLoaderWithoutCache(String fileName, Reader fileReader) {
- this.fileName = fileName;
- this.fileReader = fileReader;
- }
-
- @Override
- public Reader getReader() throws Exception {
- return fileReader;
- }
-
- @Override
- public String getFileName() {
- return fileName;
- }
-}
diff --git a/sonar-duplications/src/main/java/org/sonar/duplications/cpd/package-info.java b/sonar-duplications/src/main/java/org/sonar/duplications/cpd/package-info.java
deleted file mode 100644
index 4602f8c1b55..00000000000
--- a/sonar-duplications/src/main/java/org/sonar/duplications/cpd/package-info.java
+++ /dev/null
@@ -1,24 +0,0 @@
-/*
- * SonarQube
- * Copyright (C) 2009-2022 SonarSource SA
- * mailto:info AT sonarsource DOT com
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 3 of the License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public License
- * along with this program; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
- */
-@ParametersAreNonnullByDefault
-package org.sonar.duplications.cpd;
-
-import javax.annotation.ParametersAreNonnullByDefault;
-
diff --git a/sonar-duplications/src/main/java/org/sonar/duplications/internal/pmd/TokenizerBridge.java b/sonar-duplications/src/main/java/org/sonar/duplications/internal/pmd/TokenizerBridge.java
deleted file mode 100644
index 4c0ff81f631..00000000000
--- a/sonar-duplications/src/main/java/org/sonar/duplications/internal/pmd/TokenizerBridge.java
+++ /dev/null
@@ -1,99 +0,0 @@
-/*
- * SonarQube
- * Copyright (C) 2009-2022 SonarSource SA
- * mailto:info AT sonarsource DOT com
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 3 of the License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public License
- * along with this program; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
- */
-package org.sonar.duplications.internal.pmd;
-
-import java.io.Reader;
-import java.util.ArrayList;
-import java.util.List;
-import net.sourceforge.pmd.cpd.SourceCode;
-import net.sourceforge.pmd.cpd.TokenEntry;
-import net.sourceforge.pmd.cpd.Tokenizer;
-import net.sourceforge.pmd.cpd.Tokens;
-import org.sonar.api.batch.sensor.cpd.internal.TokensLine;
-import org.sonar.duplications.block.Block;
-import org.sonar.duplications.cpd.FileCodeLoaderWithoutCache;
-
-/**
- * Bridge, which allows to convert list of {@link TokenEntry} produced by {@link Tokenizer} into list of {@link TokensLine}s.
- */
-public class TokenizerBridge {
-
- private final Tokenizer tokenizer;
- private final PmdBlockChunker blockBuilder;
-
- public TokenizerBridge(Tokenizer tokenizer, int blockSize) {
- this.tokenizer = tokenizer;
- this.blockBuilder = new PmdBlockChunker(blockSize);
- }
-
- public List<Block> chunk(String resourceId, String fileName, Reader fileReader) {
- return blockBuilder.chunk(resourceId, chunk(fileName, fileReader));
- }
-
- public List<TokensLine> chunk(String fileName, Reader fileReader) {
- SourceCode sourceCode = new SourceCode(new FileCodeLoaderWithoutCache(fileName, fileReader));
- Tokens tokens = new Tokens();
- TokenEntry.clearImages();
- try {
- tokenizer.tokenize(sourceCode, tokens);
- } catch (RuntimeException e) {
- throw e;
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
- TokenEntry.clearImages();
- return convert(tokens.getTokens());
- }
-
- /**
- * We expect that implementation of {@link Tokenizer} is correct:
- * tokens ordered by occurrence in source code and last token is EOF.
- */
- public static List<TokensLine> convert(List<TokenEntry> tokens) {
- List<TokensLine> result = new ArrayList<>();
- StringBuilder sb = new StringBuilder();
- int startLine = Integer.MIN_VALUE;
- int startIndex = 0;
- int currentIndex = 0;
- for (TokenEntry token : tokens) {
- if (token != TokenEntry.EOF) {
- String value = token.getValue();
- int line = token.getBeginLine();
- if (line != startLine) {
- addNewTokensLine(result, startIndex, currentIndex, startLine, sb);
- startIndex = currentIndex + 1;
- startLine = line;
- }
- currentIndex++;
- sb.append(value);
- }
- }
- addNewTokensLine(result, startIndex, currentIndex, startLine, sb);
- return result;
- }
-
- private static void addNewTokensLine(List<TokensLine> result, int startUnit, int endUnit, int startLine, StringBuilder sb) {
- if (sb.length() != 0) {
- result.add(new TokensLine(startUnit, endUnit, startLine, sb.toString()));
- sb.setLength(0);
- }
- }
-
-}