Include jsoup library for modifying bootstap page DOM (#9274)

3 files changed, 169 insertions, 0 deletions

diff --git a/src/org/jsoup/examples/HtmlToPlainText.java b/src/org/jsoup/examples/HtmlToPlainText.java
new file mode 100644
index 0000000000..8f563e9608
--- /dev/null
+++ b/src/org/jsoup/examples/HtmlToPlainText.java
@@ -0,0 +1,109 @@
+package org.jsoup.examples;
+
+import org.jsoup.Jsoup;
+import org.jsoup.helper.StringUtil;
+import org.jsoup.helper.Validate;
+import org.jsoup.nodes.Document;
+import org.jsoup.nodes.Element;
+import org.jsoup.nodes.Node;
+import org.jsoup.nodes.TextNode;
+import org.jsoup.select.NodeTraversor;
+import org.jsoup.select.NodeVisitor;
+
+import java.io.IOException;
+
+/**
+ * HTML to plain-text. This example program demonstrates the use of jsoup to convert HTML input to lightly-formatted
+ * plain-text. That is divergent from the general goal of jsoup's .text() methods, which is to get clean data from a
+ * scrape.
+ * <p/>
+ * Note that this is a fairly simplistic formatter -- for real world use you'll want to embrace and extend.
+ *
+ * @author Jonathan Hedley, jonathan@hedley.net
+ */
+public class HtmlToPlainText {
+    public static void main(String... args) throws IOException {
+        Validate.isTrue(args.length == 1, "usage: supply url to fetch");
+        String url = args[0];
+
+        // fetch the specified URL and parse to a HTML DOM
+        Document doc = Jsoup.connect(url).get();
+
+        HtmlToPlainText formatter = new HtmlToPlainText();
+        String plainText = formatter.getPlainText(doc);
+        System.out.println(plainText);
+    }
+
+    /**
+     * Format an Element to plain-text
+     * @param element the root element to format
+     * @return formatted text
+     */
+    public String getPlainText(Element element) {
+        FormattingVisitor formatter = new FormattingVisitor();
+        NodeTraversor traversor = new NodeTraversor(formatter);
+        traversor.traverse(element); // walk the DOM, and call .head() and .tail() for each node
+
+        return formatter.toString();
+    }
+
+    // the formatting rules, implemented in a breadth-first DOM traverse
+    private class FormattingVisitor implements NodeVisitor {
+        private static final int maxWidth = 80;
+        private int width = 0;
+        private StringBuilder accum = new StringBuilder(); // holds the accumulated text
+
+        // hit when the node is first seen
+        public void head(Node node, int depth) {
+            String name = node.nodeName();
+            if (node instanceof TextNode)
+                append(((TextNode) node).text()); // TextNodes carry all user-readable text in the DOM.
+            else if (name.equals("li"))
+                append("\n * ");
+        }
+
+        // hit when all of the node's children (if any) have been visited
+        public void tail(Node node, int depth) {
+            String name = node.nodeName();
+            if (name.equals("br"))
+                append("\n");
+            else if (StringUtil.in(name, "p", "h1", "h2", "h3", "h4", "h5"))
+                append("\n\n");
+            else if (name.equals("a"))
+                append(String.format(" <%s>", node.absUrl("href")));
+        }
+
+        // appends text to the string builder with a simple word wrap method
+        private void append(String text) {
+            if (text.startsWith("\n"))
+                width = 0; // reset counter if starts with a newline. only from formats above, not in natural text
+            if (text.equals(" ") &&
+                    (accum.length() == 0 || StringUtil.in(accum.substring(accum.length() - 1), " ", "\n")))
+                return; // don't accumulate long runs of empty spaces
+
+            if (text.length() + width > maxWidth) { // won't fit, needs to wrap
+                String words[] = text.split("\\s+");
+                for (int i = 0; i < words.length; i++) {
+                    String word = words[i];
+                    boolean last = i == words.length - 1;
+                    if (!last) // insert a space if not the last word
+                        word = word + " ";
+                    if (word.length() + width > maxWidth) { // wrap and reset counter
+                        accum.append("\n").append(word);
+                        width = word.length();
+                    } else {
+                        accum.append(word);
+                        width += word.length();
+                    }
+                }
+            } else { // fits as is, without need to wrap text
+                accum.append(text);
+                width += text.length();
+            }
+        }
+
+        public String toString() {
+            return accum.toString();
+        }
+    }
+}
diff --git a/src/org/jsoup/examples/ListLinks.java b/src/org/jsoup/examples/ListLinks.java
new file mode 100644
index 0000000000..64b29ba107
--- /dev/null
+++ b/src/org/jsoup/examples/ListLinks.java
@@ -0,0 +1,56 @@
+package org.jsoup.examples;
+
+import org.jsoup.Jsoup;
+import org.jsoup.helper.Validate;
+import org.jsoup.nodes.Document;
+import org.jsoup.nodes.Element;
+import org.jsoup.select.Elements;
+
+import java.io.IOException;
+
+/**
+ * Example program to list links from a URL.
+ */
+public class ListLinks {
+    public static void main(String[] args) throws IOException {
+        Validate.isTrue(args.length == 1, "usage: supply url to fetch");
+        String url = args[0];
+        print("Fetching %s...", url);
+
+        Document doc = Jsoup.connect(url).get();
+        Elements links = doc.select("a[href]");
+        Elements media = doc.select("[src]");
+        Elements imports = doc.select("link[href]");
+
+        print("\nMedia: (%d)", media.size());
+        for (Element src : media) {
+            if (src.tagName().equals("img"))
+                print(" * %s: <%s> %sx%s (%s)",
+                        src.tagName(), src.attr("abs:src"), src.attr("width"), src.attr("height"),
+                        trim(src.attr("alt"), 20));
+            else
+                print(" * %s: <%s>", src.tagName(), src.attr("abs:src"));
+        }
+
+        print("\nImports: (%d)", imports.size());
+        for (Element link : imports) {
+            print(" * %s <%s> (%s)", link.tagName(),link.attr("abs:href"), link.attr("rel"));
+        }
+
+        print("\nLinks: (%d)", links.size());
+        for (Element link : links) {
+            print(" * a: <%s>  (%s)", link.attr("abs:href"), trim(link.text(), 35));
+        }
+    }
+
+    private static void print(String msg, Object... args) {
+        System.out.println(String.format(msg, args));
+    }
+
+    private static String trim(String s, int width) {
+        if (s.length() > width)
+            return s.substring(0, width-1) + ".";
+        else
+            return s;
+    }
+}
diff --git a/src/org/jsoup/examples/package-info.java b/src/org/jsoup/examples/package-info.java
new file mode 100644
index 0000000000..c312f430d4
--- /dev/null
+++ b/src/org/jsoup/examples/package-info.java
@@ -0,0 +1,4 @@
+/**
+ Contains example programs and use of jsoup. See the <a href="http://jsoup.org/cookbook/">jsoup cookbook</a>.
+ */
+package org.jsoup.examples;
\ No newline at end of file