config.action_mailer.perform_deliveries = false
config.gem 'rubytree', :lib => 'tree'
+ config.gem 'coderay', :version => '~>0.9.7'
# Load any local configuration that is kept out of source control
# (e.g. gems, patches).
--- /dev/null
+--- !ruby/object:Gem::Specification \r
+name: coderay\r
+version: !ruby/object:Gem::Version \r
+ hash: 53\r
+ prerelease: false\r
+ segments: \r
+ - 0\r
+ - 9\r
+ - 7\r
+ version: 0.9.7\r
+platform: ruby\r
+authors: \r
+- murphy\r
+autorequire: \r
+bindir: bin\r
+cert_chain: []\r
+\r
+date: 2011-01-15 00:00:00 +01:00\r
+default_executable: \r
+dependencies: []\r
+\r
+description: |\r
+ Fast and easy syntax highlighting for selected languages, written in Ruby.\r
+ Comes with RedCloth integration and LOC counter.\r
+\r
+email: murphy@rubychan.de\r
+executables: \r
+- coderay\r
+- coderay_stylesheet\r
+extensions: []\r
+\r
+extra_rdoc_files: \r
+- lib/README\r
+- FOLDERS\r
+files: \r
+- ./lib/coderay/duo.rb\r
+- ./lib/coderay/encoder.rb\r
+- ./lib/coderay/encoders/_map.rb\r
+- ./lib/coderay/encoders/comment_filter.rb\r
+- ./lib/coderay/encoders/count.rb\r
+- ./lib/coderay/encoders/debug.rb\r
+- ./lib/coderay/encoders/div.rb\r
+- ./lib/coderay/encoders/filter.rb\r
+- ./lib/coderay/encoders/html/css.rb\r
+- ./lib/coderay/encoders/html/numerization.rb\r
+- ./lib/coderay/encoders/html/output.rb\r
+- ./lib/coderay/encoders/html.rb\r
+- ./lib/coderay/encoders/json.rb\r
+- ./lib/coderay/encoders/lines_of_code.rb\r
+- ./lib/coderay/encoders/null.rb\r
+- ./lib/coderay/encoders/page.rb\r
+- ./lib/coderay/encoders/span.rb\r
+- ./lib/coderay/encoders/statistic.rb\r
+- ./lib/coderay/encoders/term.rb\r
+- ./lib/coderay/encoders/text.rb\r
+- ./lib/coderay/encoders/token_class_filter.rb\r
+- ./lib/coderay/encoders/xml.rb\r
+- ./lib/coderay/encoders/yaml.rb\r
+- ./lib/coderay/for_redcloth.rb\r
+- ./lib/coderay/helpers/file_type.rb\r
+- ./lib/coderay/helpers/gzip_simple.rb\r
+- ./lib/coderay/helpers/plugin.rb\r
+- ./lib/coderay/helpers/word_list.rb\r
+- ./lib/coderay/scanner.rb\r
+- ./lib/coderay/scanners/_map.rb\r
+- ./lib/coderay/scanners/c.rb\r
+- ./lib/coderay/scanners/cpp.rb\r
+- ./lib/coderay/scanners/css.rb\r
+- ./lib/coderay/scanners/debug.rb\r
+- ./lib/coderay/scanners/delphi.rb\r
+- ./lib/coderay/scanners/diff.rb\r
+- ./lib/coderay/scanners/groovy.rb\r
+- ./lib/coderay/scanners/html.rb\r
+- ./lib/coderay/scanners/java/builtin_types.rb\r
+- ./lib/coderay/scanners/java.rb\r
+- ./lib/coderay/scanners/java_script-0.9.6.rb\r
+- ./lib/coderay/scanners/java_script.rb\r
+- ./lib/coderay/scanners/json.rb\r
+- ./lib/coderay/scanners/nitro_xhtml.rb\r
+- ./lib/coderay/scanners/php.rb\r
+- ./lib/coderay/scanners/plaintext.rb\r
+- ./lib/coderay/scanners/python.rb\r
+- ./lib/coderay/scanners/rhtml.rb\r
+- ./lib/coderay/scanners/ruby/patterns.rb\r
+- ./lib/coderay/scanners/ruby.rb\r
+- ./lib/coderay/scanners/scheme.rb\r
+- ./lib/coderay/scanners/sql.rb\r
+- ./lib/coderay/scanners/xml.rb\r
+- ./lib/coderay/scanners/yaml.rb\r
+- ./lib/coderay/style.rb\r
+- ./lib/coderay/styles/_map.rb\r
+- ./lib/coderay/styles/cycnus.rb\r
+- ./lib/coderay/styles/murphy.rb\r
+- ./lib/coderay/token_classes.rb\r
+- ./lib/coderay/tokens.rb\r
+- ./lib/coderay.rb\r
+- ./Rakefile\r
+- ./test/functional/basic.rb\r
+- ./test/functional/basic.rbc\r
+- ./test/functional/for_redcloth.rb\r
+- ./test/functional/for_redcloth.rbc\r
+- ./test/functional/load_plugin_scanner.rb\r
+- ./test/functional/load_plugin_scanner.rbc\r
+- ./test/functional/suite.rb\r
+- ./test/functional/suite.rbc\r
+- ./test/functional/vhdl.rb\r
+- ./test/functional/vhdl.rbc\r
+- ./test/functional/word_list.rb\r
+- ./test/functional/word_list.rbc\r
+- ./lib/README\r
+- ./LICENSE\r
+- lib/README\r
+- FOLDERS\r
+- bin/coderay\r
+- bin/coderay_stylesheet\r
+has_rdoc: true\r
+homepage: http://coderay.rubychan.de\r
+licenses: []\r
+\r
+post_install_message: \r
+rdoc_options: \r
+- -SNw2\r
+- -mlib/README\r
+- -t CodeRay Documentation\r
+require_paths: \r
+- lib\r
+required_ruby_version: !ruby/object:Gem::Requirement \r
+ none: false\r
+ requirements: \r
+ - - ">="\r
+ - !ruby/object:Gem::Version \r
+ hash: 51\r
+ segments: \r
+ - 1\r
+ - 8\r
+ - 2\r
+ version: 1.8.2\r
+required_rubygems_version: !ruby/object:Gem::Requirement \r
+ none: false\r
+ requirements: \r
+ - - ">="\r
+ - !ruby/object:Gem::Version \r
+ hash: 3\r
+ segments: \r
+ - 0\r
+ version: "0"\r
+requirements: []\r
+\r
+rubyforge_project: coderay\r
+rubygems_version: 1.3.7\r
+signing_key: \r
+specification_version: 3\r
+summary: Fast syntax highlighting for selected languages.\r
+test_files: \r
+- ./test/functional/suite.rb\r
--- /dev/null
+= CodeRay - Trunk folder structure\r
+\r
+== bench - Benchmarking system\r
+\r
+All benchmarking stuff goes here.\r
+\r
+Test inputs are stored in files named <code>example.<lang></code>.\r
+Test outputs go to <code>bench/test.<encoder-default-file-extension></code>.\r
+\r
+Run <code>bench/bench.rb</code> to get a usage description.\r
+\r
+Run <code>rake bench</code> to perform an example benchmark.\r
+\r
+\r
+== bin - Scripts\r
+\r
+Executional files for CodeRay.\r
+\r
+\r
+== demo - Demos and functional tests\r
+\r
+Demonstrational scripts to show of CodeRay's features.\r
+\r
+Run them as functional tests with <code>rake test:demos</code>.\r
+\r
+\r
+== etc - Lots of stuff\r
+\r
+Some addidtional files for CodeRay, mainly graphics and Vim scripts.\r
+\r
+\r
+== gem_server - Gem output folder\r
+\r
+For <code>rake gem</code>.\r
+\r
+\r
+== lib - CodeRay library code\r
+\r
+This is the base directory for the CodeRay library.\r
+\r
+\r
+== rake_helpers - Rake helper libraries\r
+\r
+Some files to enhance Rake, including the Autumnal Rdoc template and some scripts.\r
+\r
+\r
+== test - Tests\r
+\r
+Tests for the scanners.\r
+\r
+Each language has its own subfolder and sub-suite.\r
+\r
+Run with <code>rake test</code>.\r
--- /dev/null
+ GNU LESSER GENERAL PUBLIC LICENSE
+ Version 2.1, February 1999
+
+ Copyright (C) 1991, 1999 Free Software Foundation, Inc.
+ 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+[This is the first released version of the Lesser GPL. It also counts
+ as the successor of the GNU Library Public License, version 2, hence
+ the version number 2.1.]
+
+ Preamble
+
+ The licenses for most software are designed to take away your
+freedom to share and change it. By contrast, the GNU General Public
+Licenses are intended to guarantee your freedom to share and change
+free software--to make sure the software is free for all its users.
+
+ This license, the Lesser General Public License, applies to some
+specially designated software packages--typically libraries--of the
+Free Software Foundation and other authors who decide to use it. You
+can use it too, but we suggest you first think carefully about whether
+this license or the ordinary General Public License is the better
+strategy to use in any particular case, based on the explanations below.
+
+ When we speak of free software, we are referring to freedom of use,
+not price. Our General Public Licenses are designed to make sure that
+you have the freedom to distribute copies of free software (and charge
+for this service if you wish); that you receive source code or can get
+it if you want it; that you can change the software and use pieces of
+it in new free programs; and that you are informed that you can do
+these things.
+
+ To protect your rights, we need to make restrictions that forbid
+distributors to deny you these rights or to ask you to surrender these
+rights. These restrictions translate to certain responsibilities for
+you if you distribute copies of the library or if you modify it.
+
+ For example, if you distribute copies of the library, whether gratis
+or for a fee, you must give the recipients all the rights that we gave
+you. You must make sure that they, too, receive or can get the source
+code. If you link other code with the library, you must provide
+complete object files to the recipients, so that they can relink them
+with the library after making changes to the library and recompiling
+it. And you must show them these terms so they know their rights.
+
+ We protect your rights with a two-step method: (1) we copyright the
+library, and (2) we offer you this license, which gives you legal
+permission to copy, distribute and/or modify the library.
+
+ To protect each distributor, we want to make it very clear that
+there is no warranty for the free library. Also, if the library is
+modified by someone else and passed on, the recipients should know
+that what they have is not the original version, so that the original
+author's reputation will not be affected by problems that might be
+introduced by others.
+\f
+ Finally, software patents pose a constant threat to the existence of
+any free program. We wish to make sure that a company cannot
+effectively restrict the users of a free program by obtaining a
+restrictive license from a patent holder. Therefore, we insist that
+any patent license obtained for a version of the library must be
+consistent with the full freedom of use specified in this license.
+
+ Most GNU software, including some libraries, is covered by the
+ordinary GNU General Public License. This license, the GNU Lesser
+General Public License, applies to certain designated libraries, and
+is quite different from the ordinary General Public License. We use
+this license for certain libraries in order to permit linking those
+libraries into non-free programs.
+
+ When a program is linked with a library, whether statically or using
+a shared library, the combination of the two is legally speaking a
+combined work, a derivative of the original library. The ordinary
+General Public License therefore permits such linking only if the
+entire combination fits its criteria of freedom. The Lesser General
+Public License permits more lax criteria for linking other code with
+the library.
+
+ We call this license the "Lesser" General Public License because it
+does Less to protect the user's freedom than the ordinary General
+Public License. It also provides other free software developers Less
+of an advantage over competing non-free programs. These disadvantages
+are the reason we use the ordinary General Public License for many
+libraries. However, the Lesser license provides advantages in certain
+special circumstances.
+
+ For example, on rare occasions, there may be a special need to
+encourage the widest possible use of a certain library, so that it becomes
+a de-facto standard. To achieve this, non-free programs must be
+allowed to use the library. A more frequent case is that a free
+library does the same job as widely used non-free libraries. In this
+case, there is little to gain by limiting the free library to free
+software only, so we use the Lesser General Public License.
+
+ In other cases, permission to use a particular library in non-free
+programs enables a greater number of people to use a large body of
+free software. For example, permission to use the GNU C Library in
+non-free programs enables many more people to use the whole GNU
+operating system, as well as its variant, the GNU/Linux operating
+system.
+
+ Although the Lesser General Public License is Less protective of the
+users' freedom, it does ensure that the user of a program that is
+linked with the Library has the freedom and the wherewithal to run
+that program using a modified version of the Library.
+
+ The precise terms and conditions for copying, distribution and
+modification follow. Pay close attention to the difference between a
+"work based on the library" and a "work that uses the library". The
+former contains code derived from the library, whereas the latter must
+be combined with the library in order to run.
+\f
+ GNU LESSER GENERAL PUBLIC LICENSE
+ TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
+
+ 0. This License Agreement applies to any software library or other
+program which contains a notice placed by the copyright holder or
+other authorized party saying it may be distributed under the terms of
+this Lesser General Public License (also called "this License").
+Each licensee is addressed as "you".
+
+ A "library" means a collection of software functions and/or data
+prepared so as to be conveniently linked with application programs
+(which use some of those functions and data) to form executables.
+
+ The "Library", below, refers to any such software library or work
+which has been distributed under these terms. A "work based on the
+Library" means either the Library or any derivative work under
+copyright law: that is to say, a work containing the Library or a
+portion of it, either verbatim or with modifications and/or translated
+straightforwardly into another language. (Hereinafter, translation is
+included without limitation in the term "modification".)
+
+ "Source code" for a work means the preferred form of the work for
+making modifications to it. For a library, complete source code means
+all the source code for all modules it contains, plus any associated
+interface definition files, plus the scripts used to control compilation
+and installation of the library.
+
+ Activities other than copying, distribution and modification are not
+covered by this License; they are outside its scope. The act of
+running a program using the Library is not restricted, and output from
+such a program is covered only if its contents constitute a work based
+on the Library (independent of the use of the Library in a tool for
+writing it). Whether that is true depends on what the Library does
+and what the program that uses the Library does.
+
+ 1. You may copy and distribute verbatim copies of the Library's
+complete source code as you receive it, in any medium, provided that
+you conspicuously and appropriately publish on each copy an
+appropriate copyright notice and disclaimer of warranty; keep intact
+all the notices that refer to this License and to the absence of any
+warranty; and distribute a copy of this License along with the
+Library.
+
+ You may charge a fee for the physical act of transferring a copy,
+and you may at your option offer warranty protection in exchange for a
+fee.
+\f
+ 2. You may modify your copy or copies of the Library or any portion
+of it, thus forming a work based on the Library, and copy and
+distribute such modifications or work under the terms of Section 1
+above, provided that you also meet all of these conditions:
+
+ a) The modified work must itself be a software library.
+
+ b) You must cause the files modified to carry prominent notices
+ stating that you changed the files and the date of any change.
+
+ c) You must cause the whole of the work to be licensed at no
+ charge to all third parties under the terms of this License.
+
+ d) If a facility in the modified Library refers to a function or a
+ table of data to be supplied by an application program that uses
+ the facility, other than as an argument passed when the facility
+ is invoked, then you must make a good faith effort to ensure that,
+ in the event an application does not supply such function or
+ table, the facility still operates, and performs whatever part of
+ its purpose remains meaningful.
+
+ (For example, a function in a library to compute square roots has
+ a purpose that is entirely well-defined independent of the
+ application. Therefore, Subsection 2d requires that any
+ application-supplied function or table used by this function must
+ be optional: if the application does not supply it, the square
+ root function must still compute square roots.)
+
+These requirements apply to the modified work as a whole. If
+identifiable sections of that work are not derived from the Library,
+and can be reasonably considered independent and separate works in
+themselves, then this License, and its terms, do not apply to those
+sections when you distribute them as separate works. But when you
+distribute the same sections as part of a whole which is a work based
+on the Library, the distribution of the whole must be on the terms of
+this License, whose permissions for other licensees extend to the
+entire whole, and thus to each and every part regardless of who wrote
+it.
+
+Thus, it is not the intent of this section to claim rights or contest
+your rights to work written entirely by you; rather, the intent is to
+exercise the right to control the distribution of derivative or
+collective works based on the Library.
+
+In addition, mere aggregation of another work not based on the Library
+with the Library (or with a work based on the Library) on a volume of
+a storage or distribution medium does not bring the other work under
+the scope of this License.
+
+ 3. You may opt to apply the terms of the ordinary GNU General Public
+License instead of this License to a given copy of the Library. To do
+this, you must alter all the notices that refer to this License, so
+that they refer to the ordinary GNU General Public License, version 2,
+instead of to this License. (If a newer version than version 2 of the
+ordinary GNU General Public License has appeared, then you can specify
+that version instead if you wish.) Do not make any other change in
+these notices.
+\f
+ Once this change is made in a given copy, it is irreversible for
+that copy, so the ordinary GNU General Public License applies to all
+subsequent copies and derivative works made from that copy.
+
+ This option is useful when you wish to copy part of the code of
+the Library into a program that is not a library.
+
+ 4. You may copy and distribute the Library (or a portion or
+derivative of it, under Section 2) in object code or executable form
+under the terms of Sections 1 and 2 above provided that you accompany
+it with the complete corresponding machine-readable source code, which
+must be distributed under the terms of Sections 1 and 2 above on a
+medium customarily used for software interchange.
+
+ If distribution of object code is made by offering access to copy
+from a designated place, then offering equivalent access to copy the
+source code from the same place satisfies the requirement to
+distribute the source code, even though third parties are not
+compelled to copy the source along with the object code.
+
+ 5. A program that contains no derivative of any portion of the
+Library, but is designed to work with the Library by being compiled or
+linked with it, is called a "work that uses the Library". Such a
+work, in isolation, is not a derivative work of the Library, and
+therefore falls outside the scope of this License.
+
+ However, linking a "work that uses the Library" with the Library
+creates an executable that is a derivative of the Library (because it
+contains portions of the Library), rather than a "work that uses the
+library". The executable is therefore covered by this License.
+Section 6 states terms for distribution of such executables.
+
+ When a "work that uses the Library" uses material from a header file
+that is part of the Library, the object code for the work may be a
+derivative work of the Library even though the source code is not.
+Whether this is true is especially significant if the work can be
+linked without the Library, or if the work is itself a library. The
+threshold for this to be true is not precisely defined by law.
+
+ If such an object file uses only numerical parameters, data
+structure layouts and accessors, and small macros and small inline
+functions (ten lines or less in length), then the use of the object
+file is unrestricted, regardless of whether it is legally a derivative
+work. (Executables containing this object code plus portions of the
+Library will still fall under Section 6.)
+
+ Otherwise, if the work is a derivative of the Library, you may
+distribute the object code for the work under the terms of Section 6.
+Any executables containing that work also fall under Section 6,
+whether or not they are linked directly with the Library itself.
+\f
+ 6. As an exception to the Sections above, you may also combine or
+link a "work that uses the Library" with the Library to produce a
+work containing portions of the Library, and distribute that work
+under terms of your choice, provided that the terms permit
+modification of the work for the customer's own use and reverse
+engineering for debugging such modifications.
+
+ You must give prominent notice with each copy of the work that the
+Library is used in it and that the Library and its use are covered by
+this License. You must supply a copy of this License. If the work
+during execution displays copyright notices, you must include the
+copyright notice for the Library among them, as well as a reference
+directing the user to the copy of this License. Also, you must do one
+of these things:
+
+ a) Accompany the work with the complete corresponding
+ machine-readable source code for the Library including whatever
+ changes were used in the work (which must be distributed under
+ Sections 1 and 2 above); and, if the work is an executable linked
+ with the Library, with the complete machine-readable "work that
+ uses the Library", as object code and/or source code, so that the
+ user can modify the Library and then relink to produce a modified
+ executable containing the modified Library. (It is understood
+ that the user who changes the contents of definitions files in the
+ Library will not necessarily be able to recompile the application
+ to use the modified definitions.)
+
+ b) Use a suitable shared library mechanism for linking with the
+ Library. A suitable mechanism is one that (1) uses at run time a
+ copy of the library already present on the user's computer system,
+ rather than copying library functions into the executable, and (2)
+ will operate properly with a modified version of the library, if
+ the user installs one, as long as the modified version is
+ interface-compatible with the version that the work was made with.
+
+ c) Accompany the work with a written offer, valid for at
+ least three years, to give the same user the materials
+ specified in Subsection 6a, above, for a charge no more
+ than the cost of performing this distribution.
+
+ d) If distribution of the work is made by offering access to copy
+ from a designated place, offer equivalent access to copy the above
+ specified materials from the same place.
+
+ e) Verify that the user has already received a copy of these
+ materials or that you have already sent this user a copy.
+
+ For an executable, the required form of the "work that uses the
+Library" must include any data and utility programs needed for
+reproducing the executable from it. However, as a special exception,
+the materials to be distributed need not include anything that is
+normally distributed (in either source or binary form) with the major
+components (compiler, kernel, and so on) of the operating system on
+which the executable runs, unless that component itself accompanies
+the executable.
+
+ It may happen that this requirement contradicts the license
+restrictions of other proprietary libraries that do not normally
+accompany the operating system. Such a contradiction means you cannot
+use both them and the Library together in an executable that you
+distribute.
+\f
+ 7. You may place library facilities that are a work based on the
+Library side-by-side in a single library together with other library
+facilities not covered by this License, and distribute such a combined
+library, provided that the separate distribution of the work based on
+the Library and of the other library facilities is otherwise
+permitted, and provided that you do these two things:
+
+ a) Accompany the combined library with a copy of the same work
+ based on the Library, uncombined with any other library
+ facilities. This must be distributed under the terms of the
+ Sections above.
+
+ b) Give prominent notice with the combined library of the fact
+ that part of it is a work based on the Library, and explaining
+ where to find the accompanying uncombined form of the same work.
+
+ 8. You may not copy, modify, sublicense, link with, or distribute
+the Library except as expressly provided under this License. Any
+attempt otherwise to copy, modify, sublicense, link with, or
+distribute the Library is void, and will automatically terminate your
+rights under this License. However, parties who have received copies,
+or rights, from you under this License will not have their licenses
+terminated so long as such parties remain in full compliance.
+
+ 9. You are not required to accept this License, since you have not
+signed it. However, nothing else grants you permission to modify or
+distribute the Library or its derivative works. These actions are
+prohibited by law if you do not accept this License. Therefore, by
+modifying or distributing the Library (or any work based on the
+Library), you indicate your acceptance of this License to do so, and
+all its terms and conditions for copying, distributing or modifying
+the Library or works based on it.
+
+ 10. Each time you redistribute the Library (or any work based on the
+Library), the recipient automatically receives a license from the
+original licensor to copy, distribute, link with or modify the Library
+subject to these terms and conditions. You may not impose any further
+restrictions on the recipients' exercise of the rights granted herein.
+You are not responsible for enforcing compliance by third parties with
+this License.
+\f
+ 11. If, as a consequence of a court judgment or allegation of patent
+infringement or for any other reason (not limited to patent issues),
+conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License. If you cannot
+distribute so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you
+may not distribute the Library at all. For example, if a patent
+license would not permit royalty-free redistribution of the Library by
+all those who receive copies directly or indirectly through you, then
+the only way you could satisfy both it and this License would be to
+refrain entirely from distribution of the Library.
+
+If any portion of this section is held invalid or unenforceable under any
+particular circumstance, the balance of the section is intended to apply,
+and the section as a whole is intended to apply in other circumstances.
+
+It is not the purpose of this section to induce you to infringe any
+patents or other property right claims or to contest validity of any
+such claims; this section has the sole purpose of protecting the
+integrity of the free software distribution system which is
+implemented by public license practices. Many people have made
+generous contributions to the wide range of software distributed
+through that system in reliance on consistent application of that
+system; it is up to the author/donor to decide if he or she is willing
+to distribute software through any other system and a licensee cannot
+impose that choice.
+
+This section is intended to make thoroughly clear what is believed to
+be a consequence of the rest of this License.
+
+ 12. If the distribution and/or use of the Library is restricted in
+certain countries either by patents or by copyrighted interfaces, the
+original copyright holder who places the Library under this License may add
+an explicit geographical distribution limitation excluding those countries,
+so that distribution is permitted only in or among countries not thus
+excluded. In such case, this License incorporates the limitation as if
+written in the body of this License.
+
+ 13. The Free Software Foundation may publish revised and/or new
+versions of the Lesser General Public License from time to time.
+Such new versions will be similar in spirit to the present version,
+but may differ in detail to address new problems or concerns.
+
+Each version is given a distinguishing version number. If the Library
+specifies a version number of this License which applies to it and
+"any later version", you have the option of following the terms and
+conditions either of that version or of any later version published by
+the Free Software Foundation. If the Library does not specify a
+license version number, you may choose any version ever published by
+the Free Software Foundation.
+\f
+ 14. If you wish to incorporate parts of the Library into other free
+programs whose distribution conditions are incompatible with these,
+write to the author to ask for permission. For software which is
+copyrighted by the Free Software Foundation, write to the Free
+Software Foundation; we sometimes make exceptions for this. Our
+decision will be guided by the two goals of preserving the free status
+of all derivatives of our free software and of promoting the sharing
+and reuse of software generally.
+
+ NO WARRANTY
+
+ 15. BECAUSE THE LIBRARY IS LICENSED FREE OF CHARGE, THERE IS NO
+WARRANTY FOR THE LIBRARY, TO THE EXTENT PERMITTED BY APPLICABLE LAW.
+EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR
+OTHER PARTIES PROVIDE THE LIBRARY "AS IS" WITHOUT WARRANTY OF ANY
+KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE
+LIBRARY IS WITH YOU. SHOULD THE LIBRARY PROVE DEFECTIVE, YOU ASSUME
+THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
+
+ 16. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN
+WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY
+AND/OR REDISTRIBUTE THE LIBRARY AS PERMITTED ABOVE, BE LIABLE TO YOU
+FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR
+CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE
+LIBRARY (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING
+RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A
+FAILURE OF THE LIBRARY TO OPERATE WITH ANY OTHER SOFTWARE), EVEN IF
+SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH
+DAMAGES.
+
+ END OF TERMS AND CONDITIONS
+\f
+ How to Apply These Terms to Your New Libraries
+
+ If you develop a new library, and you want it to be of the greatest
+possible use to the public, we recommend making it free software that
+everyone can redistribute and change. You can do so by permitting
+redistribution under these terms (or, alternatively, under the terms of the
+ordinary General Public License).
+
+ To apply these terms, attach the following notices to the library. It is
+safest to attach them to the start of each source file to most effectively
+convey the exclusion of warranty; and each file should have at least the
+"copyright" line and a pointer to where the full notice is found.
+
+ <one line to give the library's name and a brief idea of what it does.>
+ Copyright (C) <year> <name of author>
+
+ This library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ This library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with this library; if not, write to the Free Software
+ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+
+Also add information on how to contact you by electronic and paper mail.
+
+You should also get your employer (if you work as a programmer) or your
+school, if any, to sign a "copyright disclaimer" for the library, if
+necessary. Here is a sample; alter the names:
+
+ Yoyodyne, Inc., hereby disclaims all copyright interest in the
+ library `Frob' (a library for tweaking knobs) written by James Random Hacker.
+
+ <signature of Ty Coon>, 1 April 1990
+ Ty Coon, President of Vice
+
+That's all there is to it!
+
+
--- /dev/null
+require 'rake/rdoctask'
+
+ROOT = '.'
+LIB_ROOT = File.join ROOT, 'lib'
+EXTRA_RDOC_FILES = %w(lib/README FOLDERS)
+
+task :default => :test
+
+if File.directory? 'rake_tasks'
+
+ # load rake tasks from subfolder
+ for task_file in Dir['rake_tasks/*.rake'].sort
+ load task_file
+ end
+
+else
+
+ # fallback tasks when rake_tasks folder is not present
+ desc 'Run CodeRay tests (basic)'
+ task :test do
+ ruby './test/functional/suite.rb'
+ ruby './test/functional/for_redcloth.rb'
+ end
+
+ desc 'Generate documentation for CodeRay'
+ Rake::RDocTask.new :doc do |rd|
+ rd.title = 'CodeRay Documentation'
+ rd.main = 'lib/README'
+ rd.rdoc_files.add Dir['lib']
+ rd.rdoc_files.add 'lib/README'
+ rd.rdoc_files.add 'FOLDERS'
+ rd.rdoc_dir = 'doc'
+ end
+
+end
\ No newline at end of file
--- /dev/null
+#!/usr/bin/env ruby
+# CodeRay Executable
+#
+# Version: 0.2
+# Author: murphy
+
+require 'coderay'
+
+if ARGV.empty?
+ $stderr.puts <<-USAGE
+CodeRay #{CodeRay::VERSION} (http://coderay.rubychan.de)
+
+Usage:
+ coderay file [-<format>]
+ coderay -<lang> [-<format>] [< file] [> output]
+
+Defaults:
+ lang: based on file extension
+ format: ANSI colorized output for terminal, HTML page for files
+
+Examples:
+ coderay foo.rb # colorized output to terminal, based on file extension
+ coderay foo.rb -loc # print LOC count, based on file extension and format
+ coderay foo.rb > foo.html # HTML page output to file, based on extension
+ coderay -ruby < foo.rb # colorized output to terminal, based on lang
+ coderay -ruby -loc < foo.rb # print LOC count, based on lang
+ coderay -ruby -page foo.rb # HTML page output to terminal, based on lang and format
+ coderay -ruby -page foo.rb > foo.html # HTML page output to file, based on lang and format
+ USAGE
+end
+
+first, second = ARGV
+
+def read
+ file = ARGV.grep(/^(?!-)/).last
+ if file
+ if File.exist?(file)
+ File.read file
+ else
+ $stderr.puts "No such file: #{file}"
+ end
+ else
+ $stdin.read
+ end
+end
+
+if first
+ if first[/-(\w+)/] == first
+ lang = $1
+ input = read
+ tokens = :scan
+ else
+ file = first
+ unless File.exist? file
+ $stderr.puts "No such file: #{file}"
+ exit 2
+ end
+ tokens = CodeRay.scan_file file
+ end
+else
+ $stderr.puts 'No lang/file given.'
+ exit 1
+end
+
+if second
+ if second[/-(\w+)/] == second
+ format = $1.to_sym
+ else
+ raise 'invalid format (must be -xxx)'
+ end
+else
+ if $stdout.tty?
+ format = :term
+ else
+ $stderr.puts 'No format given; setting to default (HTML Page).'
+ format = :page
+ end
+end
+
+if tokens == :scan
+ output = CodeRay::Duo[lang => format].highlight input
+else
+ output = tokens.encode format
+end
+out = $stdout
+out.puts output
--- /dev/null
+#!/usr/bin/env ruby\r
+require 'coderay'\r
+\r
+puts CodeRay::Encoders[:html]::CSS.new.stylesheet\r
--- /dev/null
+= CodeRay
+
+[- Tired of blue'n'gray? Try the original version of this documentation on
+coderay.rubychan.de[http://coderay.rubychan.de/doc/] (use Ctrl+Click to open it in its own frame.) -]
+
+== About
+CodeRay is a Ruby library for syntax highlighting.
+
+Syntax highlighting means: You put your code in, and you get it back colored;
+Keywords, strings, floats, comments - all in different colors.
+And with line numbers.
+
+*Syntax* *Highlighting*...
+* makes code easier to read and maintain
+* lets you detect syntax errors faster
+* helps you to understand the syntax of a language
+* looks nice
+* is what everybody should have on their website
+* solves all your problems and makes the girls run after you
+
+Version: 0.9.7
+Author:: murphy (Kornelius Kalnbach)
+Contact:: murphy rubychan de
+Website:: coderay.rubychan.de[http://coderay.rubychan.de]
+License:: GNU LGPL; see LICENSE file in the main directory.
+
+== Installation
+
+You need RubyGems[http://rubyforge.org/frs/?group_id=126].
+
+ % gem install coderay
+
+
+=== Dependencies
+
+CodeRay needs Ruby 1.8.6 or later. It also runs with Ruby 1.9.1+ and JRuby 1.1+.
+
+
+== Example Usage
+(Forgive me, but this is not highlighted.)
+
+ require 'coderay'
+
+ tokens = CodeRay.scan "puts 'Hello, world!'", :ruby
+ page = tokens.html :line_numbers => :inline, :wrap => :page
+ puts page
+
+
+== Documentation
+
+See CodeRay.
+
+Please report errors in this documentation to <murphy rubychan de>.
+
+
+== Credits
+
+=== Special Thanks to
+
+* licenser (Heinz N. Gies) for ending my QBasic career, inventing the Coder
+ project and the input/output plugin system.
+ CodeRay would not exist without him.
+* bovi (Daniel Bovensiepen) for helping me out on various occasions.
+
+=== Thanks to
+
+* Caleb Clausen for writing RubyLexer (see
+ http://rubyforge.org/projects/rubylexer) and lots of very interesting mail
+ traffic
+* birkenfeld (Georg Brandl) and mitsuhiku (Arnim Ronacher) for PyKleur, now pygments.
+ You guys rock!
+* Jamis Buck for writing Syntax (see http://rubyforge.org/projects/syntax)
+ I got some useful ideas from it.
+* Doug Kearns and everyone else who worked on ruby.vim - it not only helped me
+ coding CodeRay, but also gave me a wonderful target to reach for the Ruby
+ scanner.
+* everyone who uses CodeBB on http://www.rubyforen.de and http://www.python-forum.de
+* iGEL, magichisoka, manveru, WoNáDo and everyone I forgot from rubyforen.de
+* Dethix from ruby-mine.de
+* zickzackw
+* Dookie (who is no longer with us...) and Leonidas from http://www.python-forum.de
+* Andreas Schwarz for finding out that CaseIgnoringWordList was not case
+ ignoring! Such things really make you write tests.
+* closure for the first version of the Scheme scanner.
+* Stefan Walk for the first version of the JavaScript and PHP scanners.
+* Josh Goebel for another version of the JavaScript scanner, a SQL and a Diff scanner.
+* Jonathan Younger for pointing out the licence confusion caused by wrong LICENSE file.
+* Jeremy Hinegardner for finding the shebang-on-empty-file bug in FileType.
+* Charles Oliver Nutter and Yehuda Katz for helping me benchmark CodeRay on JRuby.
+* Andreas Neuhaus for pointing out a markup bug in coderay/for_redcloth.
+* 0xf30fc7 for the FileType patch concerning Delphi file extensions.
+* The folks at redmine.org - thank you for using and fixing CodeRay!
+* Keith Pitt for his SQL scanners
+* Rob Aldred for the terminal encoder
+* Trans for pointing out $DEBUG dependencies
+* Flameeyes for finding that Term::ANSIColor was obsolete
+* Etienne Massip for reporting a serious bug in JavaScript scanner
+* matz and all Ruby gods and gurus
+* The inventors of: the computer, the internet, the true color display, HTML &
+ CSS, VIM, Ruby, pizza, microwaves, guitars, scouting, programming, anime,
+ manga, coke and green ice tea.
+
+Where would we be without all those people?
+
+=== Created using
+
+* Ruby[http://ruby-lang.org/]
+* Chihiro (my Sony VAIO laptop); Henrietta (my old MacBook);
+ Triella, born Rico (my new MacBook); as well as
+ Seras and Hikari (my PCs)
+* RDE[http://homepage2.nifty.com/sakazuki/rde_e.html],
+ VIM[http://vim.org] and TextMate[http://macromates.com]
+* Subversion[http://subversion.tigris.org/]
+* Redmine[http://redmine.org/]
+* Firefox[http://www.mozilla.org/products/firefox/],
+ Firebug[http://getfirebug.com/], Safari[http://www.apple.com/safari/], and
+ Thunderbird[http://www.mozilla.org/products/thunderbird/]
+* RubyGems[http://docs.rubygems.org/] and Rake[http://rake.rubyforge.org/]
+* TortoiseSVN[http://tortoisesvn.tigris.org/] using Apache via
+ XAMPP[http://www.apachefriends.org/en/xampp.html]
+* RDoc (though I'm quite unsatisfied with it)
+* Microsoft Windows (yes, I confess!) and MacOS X
+* GNUWin32, MinGW and some other tools to make the shell under windows a bit
+ less useless
+* Term::ANSIColor[http://term-ansicolor.rubyforge.org/]
+* PLEAC[http://pleac.sourceforge.net/] code examples
+
+=== Free
+
+* As you can see, CodeRay was created under heavy use of *free* software.
+* So CodeRay is also *free*.
+* If you use CodeRay to create software, think about making this software
+ *free*, too.
+* Thanks :)
--- /dev/null
+# = CodeRay Library
+#
+# CodeRay is a Ruby library for syntax highlighting.
+#
+# I try to make CodeRay easy to use and intuitive, but at the same time fully featured, complete,
+# fast and efficient.
+#
+# See README.
+#
+# It consists mainly of
+# * the main engine: CodeRay (Scanners::Scanner, Tokens/TokenStream, Encoders::Encoder), PluginHost
+# * the scanners in CodeRay::Scanners
+# * the encoders in CodeRay::Encoders
+#
+# Here's a fancy graphic to light up this gray docu:
+#
+# http://cycnus.de/raindark/coderay/scheme.png
+#
+# == Documentation
+#
+# See CodeRay, Encoders, Scanners, Tokens.
+#
+# == Usage
+#
+# Remember you need RubyGems to use CodeRay, unless you have it in your load path. Run Ruby with
+# -rubygems option if required.
+#
+# === Highlight Ruby code in a string as html
+#
+# require 'coderay'
+# print CodeRay.scan('puts "Hello, world!"', :ruby).html
+#
+# # prints something like this:
+# puts <span class="s">"Hello, world!"</span>
+#
+#
+# === Highlight C code from a file in a html div
+#
+# require 'coderay'
+# print CodeRay.scan(File.read('ruby.h'), :c).div
+# print CodeRay.scan_file('ruby.h').html.div
+#
+# You can include this div in your page. The used CSS styles can be printed with
+#
+# % coderay_stylesheet
+#
+# === Highlight without typing too much
+#
+# If you are one of the hasty (or lazy, or extremely curious) people, just run this file:
+#
+# % ruby -rubygems /path/to/coderay/coderay.rb > example.html
+#
+# and look at the file it created in your browser.
+#
+# = CodeRay Module
+#
+# The CodeRay module provides convenience methods for the engine.
+#
+# * The +lang+ and +format+ arguments select Scanner and Encoder to use. These are
+# simply lower-case symbols, like <tt>:python</tt> or <tt>:html</tt>.
+# * All methods take an optional hash as last parameter, +options+, that is send to
+# the Encoder / Scanner.
+# * Input and language are always sorted in this order: +code+, +lang+.
+# (This is in alphabetical order, if you need a mnemonic ;)
+#
+# You should be able to highlight everything you want just using these methods;
+# so there is no need to dive into CodeRay's deep class hierarchy.
+#
+# The examples in the demo directory demonstrate common cases using this interface.
+#
+# = Basic Access Ways
+#
+# Read this to get a general view what CodeRay provides.
+#
+# == Scanning
+#
+# Scanning means analysing an input string, splitting it up into Tokens.
+# Each Token knows about what type it is: string, comment, class name, etc.
+#
+# Each +lang+ (language) has its own Scanner; for example, <tt>:ruby</tt> code is
+# handled by CodeRay::Scanners::Ruby.
+#
+# CodeRay.scan:: Scan a string in a given language into Tokens.
+# This is the most common method to use.
+# CodeRay.scan_file:: Scan a file and guess the language using FileType.
+#
+# The Tokens object you get from these methods can encode itself; see Tokens.
+#
+# == Encoding
+#
+# Encoding means compiling Tokens into an output. This can be colored HTML or
+# LaTeX, a textual statistic or just the number of non-whitespace tokens.
+#
+# Each Encoder provides output in a specific +format+, so you select Encoders via
+# formats like <tt>:html</tt> or <tt>:statistic</tt>.
+#
+# CodeRay.encode:: Scan and encode a string in a given language.
+# CodeRay.encode_tokens:: Encode the given tokens.
+# CodeRay.encode_file:: Scan a file, guess the language using FileType and encode it.
+#
+# == Streaming
+#
+# Streaming saves RAM by running Scanner and Encoder in some sort of
+# pipe mode; see TokenStream.
+#
+# CodeRay.scan_stream:: Scan in stream mode.
+#
+# == All-in-One Encoding
+#
+# CodeRay.encode:: Highlight a string with a given input and output format.
+#
+# == Instanciating
+#
+# You can use an Encoder instance to highlight multiple inputs. This way, the setup
+# for this Encoder must only be done once.
+#
+# CodeRay.encoder:: Create an Encoder instance with format and options.
+# CodeRay.scanner:: Create an Scanner instance for lang, with '' as default code.
+#
+# To make use of CodeRay.scanner, use CodeRay::Scanner::code=.
+#
+# The scanning methods provide more flexibility; we recommend to use these.
+#
+# == Reusing Scanners and Encoders
+#
+# If you want to re-use scanners and encoders (because that is faster), see
+# CodeRay::Duo for the most convenient (and recommended) interface.
+module CodeRay
+
+ $CODERAY_DEBUG ||= false
+
+ # Version: Major.Minor.Teeny[.Revision]
+ # Major: 0 for pre-stable, 1 for stable
+ # Minor: feature milestone
+ # Teeny: development state, 0 for pre-release
+ # Revision: Subversion Revision number (generated on rake gem:make)
+ VERSION = '0.9.7'
+
+ require 'coderay/tokens'
+ require 'coderay/token_classes'
+ require 'coderay/scanner'
+ require 'coderay/encoder'
+ require 'coderay/duo'
+ require 'coderay/style'
+
+
+ class << self
+
+ # Scans the given +code+ (a String) with the Scanner for +lang+.
+ #
+ # This is a simple way to use CodeRay. Example:
+ # require 'coderay'
+ # page = CodeRay.scan("puts 'Hello, world!'", :ruby).html
+ #
+ # See also demo/demo_simple.
+ def scan code, lang, options = {}, &block
+ scanner = Scanners[lang].new code, options, &block
+ scanner.tokenize
+ end
+
+ # Scans +filename+ (a path to a code file) with the Scanner for +lang+.
+ #
+ # If +lang+ is :auto or omitted, the CodeRay::FileType module is used to
+ # determine it. If it cannot find out what type it is, it uses
+ # CodeRay::Scanners::Plaintext.
+ #
+ # Calls CodeRay.scan.
+ #
+ # Example:
+ # require 'coderay'
+ # page = CodeRay.scan_file('some_c_code.c').html
+ def scan_file filename, lang = :auto, options = {}, &block
+ file = IO.read filename
+ if lang == :auto
+ require 'coderay/helpers/file_type'
+ lang = FileType.fetch filename, :plaintext, true
+ end
+ scan file, lang, options = {}, &block
+ end
+
+ # Scan the +code+ (a string) with the scanner for +lang+.
+ #
+ # Calls scan.
+ #
+ # See CodeRay.scan.
+ def scan_stream code, lang, options = {}, &block
+ options[:stream] = true
+ scan code, lang, options, &block
+ end
+
+ # Encode a string in Streaming mode.
+ #
+ # This starts scanning +code+ with the the Scanner for +lang+
+ # while encodes the output with the Encoder for +format+.
+ # +options+ will be passed to the Encoder.
+ #
+ # See CodeRay::Encoder.encode_stream
+ def encode_stream code, lang, format, options = {}
+ encoder(format, options).encode_stream code, lang, options
+ end
+
+ # Encode a string.
+ #
+ # This scans +code+ with the the Scanner for +lang+ and then
+ # encodes it with the Encoder for +format+.
+ # +options+ will be passed to the Encoder.
+ #
+ # See CodeRay::Encoder.encode
+ def encode code, lang, format, options = {}
+ encoder(format, options).encode code, lang, options
+ end
+
+ # Highlight a string into a HTML <div>.
+ #
+ # CSS styles use classes, so you have to include a stylesheet
+ # in your output.
+ #
+ # See encode.
+ def highlight code, lang, options = { :css => :class }, format = :div
+ encode code, lang, format, options
+ end
+
+ # Encode pre-scanned Tokens.
+ # Use this together with CodeRay.scan:
+ #
+ # require 'coderay'
+ #
+ # # Highlight a short Ruby code example in a HTML span
+ # tokens = CodeRay.scan '1 + 2', :ruby
+ # puts CodeRay.encode_tokens(tokens, :span)
+ #
+ def encode_tokens tokens, format, options = {}
+ encoder(format, options).encode_tokens tokens, options
+ end
+
+ # Encodes +filename+ (a path to a code file) with the Scanner for +lang+.
+ #
+ # See CodeRay.scan_file.
+ # Notice that the second argument is the output +format+, not the input language.
+ #
+ # Example:
+ # require 'coderay'
+ # page = CodeRay.encode_file 'some_c_code.c', :html
+ def encode_file filename, format, options = {}
+ tokens = scan_file filename, :auto, get_scanner_options(options)
+ encode_tokens tokens, format, options
+ end
+
+ # Highlight a file into a HTML <div>.
+ #
+ # CSS styles use classes, so you have to include a stylesheet
+ # in your output.
+ #
+ # See encode.
+ def highlight_file filename, options = { :css => :class }, format = :div
+ encode_file filename, format, options
+ end
+
+ # Finds the Encoder class for +format+ and creates an instance, passing
+ # +options+ to it.
+ #
+ # Example:
+ # require 'coderay'
+ #
+ # stats = CodeRay.encoder(:statistic)
+ # stats.encode("puts 17 + 4\n", :ruby)
+ #
+ # puts '%d out of %d tokens have the kind :integer.' % [
+ # stats.type_stats[:integer].count,
+ # stats.real_token_count
+ # ]
+ # #-> 2 out of 4 tokens have the kind :integer.
+ def encoder format, options = {}
+ Encoders[format].new options
+ end
+
+ # Finds the Scanner class for +lang+ and creates an instance, passing
+ # +options+ to it.
+ #
+ # See Scanner.new.
+ def scanner lang, options = {}
+ Scanners[lang].new '', options
+ end
+
+ # Extract the options for the scanner from the +options+ hash.
+ #
+ # Returns an empty Hash if <tt>:scanner_options</tt> is not set.
+ #
+ # This is used if a method like CodeRay.encode has to provide options
+ # for Encoder _and_ scanner.
+ def get_scanner_options options
+ options.fetch :scanner_options, {}
+ end
+
+ end
+
+ # This Exception is raised when you try to stream with something that is not
+ # capable of streaming.
+ class NotStreamableError < Exception
+ def initialize obj
+ @obj = obj
+ end
+
+ def to_s
+ '%s is not Streamable!' % @obj.class
+ end
+ end
+
+ # A dummy module that is included by subclasses of CodeRay::Scanner an CodeRay::Encoder
+ # to show that they are able to handle streams.
+ module Streamable
+ end
+
+end
+
+# Run a test script.
+if $0 == __FILE__
+ $stderr.print 'Press key to print demo.'; gets
+ # Just use this file as an example of Ruby code.
+ code = File.read(__FILE__)[/module CodeRay.*/m]
+ print CodeRay.scan(code, :ruby).html
+end
--- /dev/null
+module CodeRay
+
+ # = Duo
+ #
+ # A Duo is a convenient way to use CodeRay. You just create a Duo,
+ # giving it a lang (language of the input code) and a format (desired
+ # output format), and call Duo#highlight with the code.
+ #
+ # Duo makes it easy to re-use both scanner and encoder for a repetitive
+ # task. It also provides a very easy interface syntax:
+ #
+ # require 'coderay'
+ # CodeRay::Duo[:python, :div].highlight 'import this'
+ #
+ # Until you want to do uncommon things with CodeRay, I recommend to use
+ # this method, since it takes care of everything.
+ class Duo
+
+ attr_accessor :lang, :format, :options
+
+ # Create a new Duo, holding a lang and a format to highlight code.
+ #
+ # simple:
+ # CodeRay::Duo[:ruby, :page].highlight 'bla 42'
+ #
+ # streaming:
+ # CodeRay::Duo[:ruby, :page].highlight 'bar 23', :stream => true
+ #
+ # with options:
+ # CodeRay::Duo[:ruby, :html, :hint => :debug].highlight '????::??'
+ #
+ # alternative syntax without options:
+ # CodeRay::Duo[:ruby => :statistic].encode 'class << self; end'
+ #
+ # alternative syntax with options:
+ # CodeRay::Duo[{ :ruby => :statistic }, :do => :something].encode 'abc'
+ #
+ # The options are forwarded to scanner and encoder
+ # (see CodeRay.get_scanner_options).
+ def initialize lang = nil, format = nil, options = {}
+ if format == nil and lang.is_a? Hash and lang.size == 1
+ @lang = lang.keys.first
+ @format = lang[@lang]
+ else
+ @lang = lang
+ @format = format
+ end
+ @options = options
+ end
+
+ class << self
+ # To allow calls like Duo[:ruby, :html].highlight.
+ alias [] new
+ end
+
+ # The scanner of the duo. Only created once.
+ def scanner
+ @scanner ||= CodeRay.scanner @lang, CodeRay.get_scanner_options(@options)
+ end
+
+ # The encoder of the duo. Only created once.
+ def encoder
+ @encoder ||= CodeRay.encoder @format, @options
+ end
+
+ # Tokenize and highlight the code using +scanner+ and +encoder+.
+ #
+ # If the :stream option is set, the Duo will go into streaming mode,
+ # saving memory for the cost of time.
+ def encode code, options = { :stream => false }
+ stream = options.delete :stream
+ options = @options.merge options
+ if stream
+ encoder.encode_stream(code, @lang, options)
+ else
+ scanner.code = code
+ encoder.encode_tokens(scanner.tokenize, options)
+ end
+ end
+ alias highlight encode
+
+ end
+
+end
+
--- /dev/null
+module CodeRay
+
+ # This module holds the Encoder class and its subclasses.
+ # For example, the HTML encoder is named CodeRay::Encoders::HTML
+ # can be found in coderay/encoders/html.
+ #
+ # Encoders also provides methods and constants for the register
+ # mechanism and the [] method that returns the Encoder class
+ # belonging to the given format.
+ module Encoders
+ extend PluginHost
+ plugin_path File.dirname(__FILE__), 'encoders'
+
+ # = Encoder
+ #
+ # The Encoder base class. Together with Scanner and
+ # Tokens, it forms the highlighting triad.
+ #
+ # Encoder instances take a Tokens object and do something with it.
+ #
+ # The most common Encoder is surely the HTML encoder
+ # (CodeRay::Encoders::HTML). It highlights the code in a colorful
+ # html page.
+ # If you want the highlighted code in a div or a span instead,
+ # use its subclasses Div and Span.
+ class Encoder
+ extend Plugin
+ plugin_host Encoders
+
+ attr_reader :token_stream
+
+ class << self
+
+ # Returns if the Encoder can be used in streaming mode.
+ def streamable?
+ is_a? Streamable
+ end
+
+ # If FILE_EXTENSION isn't defined, this method returns the
+ # downcase class name instead.
+ def const_missing sym
+ if sym == :FILE_EXTENSION
+ plugin_id
+ else
+ super
+ end
+ end
+
+ end
+
+ # Subclasses are to store their default options in this constant.
+ DEFAULT_OPTIONS = { :stream => false }
+
+ # The options you gave the Encoder at creating.
+ attr_accessor :options
+
+ # Creates a new Encoder.
+ # +options+ is saved and used for all encode operations, as long
+ # as you don't overwrite it there by passing additional options.
+ #
+ # Encoder objects provide three encode methods:
+ # - encode simply takes a +code+ string and a +lang+
+ # - encode_tokens expects a +tokens+ object instead
+ # - encode_stream is like encode, but uses streaming mode.
+ #
+ # Each method has an optional +options+ parameter. These are
+ # added to the options you passed at creation.
+ def initialize options = {}
+ @options = self.class::DEFAULT_OPTIONS.merge options
+ raise "I am only the basic Encoder class. I can't encode "\
+ "anything. :( Use my subclasses." if self.class == Encoder
+ end
+
+ # Encode a Tokens object.
+ def encode_tokens tokens, options = {}
+ options = @options.merge options
+ setup options
+ compile tokens, options
+ finish options
+ end
+
+ # Encode the given +code+ after tokenizing it using the Scanner
+ # for +lang+.
+ def encode code, lang, options = {}
+ options = @options.merge options
+ scanner_options = CodeRay.get_scanner_options(options)
+ tokens = CodeRay.scan code, lang, scanner_options
+ encode_tokens tokens, options
+ end
+
+ # You can use highlight instead of encode, if that seems
+ # more clear to you.
+ alias highlight encode
+
+ # Encode the given +code+ using the Scanner for +lang+ in
+ # streaming mode.
+ def encode_stream code, lang, options = {}
+ raise NotStreamableError, self unless kind_of? Streamable
+ options = @options.merge options
+ setup options
+ scanner_options = CodeRay.get_scanner_options options
+ @token_stream =
+ CodeRay.scan_stream code, lang, scanner_options, &self
+ finish options
+ end
+
+ # Behave like a proc. The token method is converted to a proc.
+ def to_proc
+ method(:token).to_proc
+ end
+
+ # Return the default file extension for outputs of this encoder.
+ def file_extension
+ self.class::FILE_EXTENSION
+ end
+
+ protected
+
+ # Called with merged options before encoding starts.
+ # Sets @out to an empty string.
+ #
+ # See the HTML Encoder for an example of option caching.
+ def setup options
+ @out = ''
+ end
+
+ # Called with +content+ and +kind+ of the currently scanned token.
+ # For simple scanners, it's enougth to implement this method.
+ #
+ # By default, it calls text_token or block_token, depending on
+ # whether +content+ is a String.
+ def token content, kind
+ encoded_token =
+ if content.is_a? ::String
+ text_token content, kind
+ elsif content.is_a? ::Symbol
+ block_token content, kind
+ else
+ raise 'Unknown token content type: %p' % [content]
+ end
+ append_encoded_token_to_output encoded_token
+ end
+
+ def append_encoded_token_to_output encoded_token
+ @out << encoded_token if encoded_token && defined?(@out) && @out
+ end
+
+ # Called for each text token ([text, kind]), where text is a String.
+ def text_token text, kind
+ end
+
+ # Called for each block (non-text) token ([action, kind]),
+ # where +action+ is a Symbol.
+ #
+ # Calls open_token, close_token, begin_line, and end_line according to
+ # the value of +action+.
+ def block_token action, kind
+ case action
+ when :open
+ open_token kind
+ when :close
+ close_token kind
+ when :begin_line
+ begin_line kind
+ when :end_line
+ end_line kind
+ else
+ raise 'unknown block action: %p' % action
+ end
+ end
+
+ # Called for each block token at the start of the block ([:open, kind]).
+ def open_token kind
+ end
+
+ # Called for each block token end of the block ([:close, kind]).
+ def close_token kind
+ end
+
+ # Called for each line token block at the start of the line ([:begin_line, kind]).
+ def begin_line kind
+ end
+
+ # Called for each line token block at the end of the line ([:end_line, kind]).
+ def end_line kind
+ end
+
+ # Called with merged options after encoding starts.
+ # The return value is the result of encoding, typically @out.
+ def finish options
+ @out
+ end
+
+ # Do the encoding.
+ #
+ # The already created +tokens+ object must be used; it can be a
+ # TokenStream or a Tokens object.
+ if RUBY_VERSION >= '1.9'
+ def compile tokens, options
+ for text, kind in tokens
+ token text, kind
+ end
+ end
+ else
+ def compile tokens, options
+ tokens.each(&self)
+ end
+ end
+
+ end
+
+ end
+end
--- /dev/null
+module CodeRay
+module Encoders
+
+ map \
+ :loc => :lines_of_code,
+ :plain => :text,
+ :stats => :statistic,
+ :terminal => :term,
+ :tex => :latex
+
+end
+end
--- /dev/null
+($:.unshift '../..'; require 'coderay') unless defined? CodeRay
+module CodeRay
+module Encoders
+
+ load :token_class_filter
+
+ class CommentFilter < TokenClassFilter
+
+ register_for :comment_filter
+
+ DEFAULT_OPTIONS = superclass::DEFAULT_OPTIONS.merge \
+ :exclude => [:comment]
+
+ end
+
+end
+end
+
+if $0 == __FILE__
+ $VERBOSE = true
+ $: << File.join(File.dirname(__FILE__), '..')
+ eval DATA.read, nil, $0, __LINE__ + 4
+end
+
+__END__
+require 'test/unit'
+
+class CommentFilterTest < Test::Unit::TestCase
+
+ def test_filtering_comments
+ tokens = CodeRay.scan <<-RUBY, :ruby
+#!/usr/bin/env ruby
+# a minimal Ruby program
+puts "Hello world!"
+ RUBY
+ assert_equal <<-RUBY_FILTERED, tokens.comment_filter.text
+#!/usr/bin/env ruby
+
+puts "Hello world!"
+ RUBY_FILTERED
+ end
+
+end
\ No newline at end of file
--- /dev/null
+module CodeRay
+module Encoders
+
+ class Count < Encoder
+
+ include Streamable
+ register_for :count
+
+ protected
+
+ def setup options
+ @out = 0
+ end
+
+ def token text, kind
+ @out += 1
+ end
+ end
+
+end
+end
--- /dev/null
+module CodeRay
+module Encoders
+
+ # = Debug Encoder
+ #
+ # Fast encoder producing simple debug output.
+ #
+ # It is readable and diff-able and is used for testing.
+ #
+ # You cannot fully restore the tokens information from the
+ # output, because consecutive :space tokens are merged.
+ # Use Tokens#dump for caching purposes.
+ class Debug < Encoder
+
+ include Streamable
+ register_for :debug
+
+ FILE_EXTENSION = 'raydebug'
+
+ protected
+ def text_token text, kind
+ if kind == :space
+ text
+ else
+ text = text.gsub(/[)\\]/, '\\\\\0') # escape ) and \
+ "#{kind}(#{text})"
+ end
+ end
+
+ def open_token kind
+ "#{kind}<"
+ end
+
+ def close_token kind
+ ">"
+ end
+
+ def begin_line kind
+ "#{kind}["
+ end
+
+ def end_line kind
+ "]"
+ end
+
+ end
+
+end
+end
--- /dev/null
+module CodeRay
+module Encoders
+
+ load :html
+
+ class Div < HTML
+
+ FILE_EXTENSION = 'div.html'
+
+ register_for :div
+
+ DEFAULT_OPTIONS = HTML::DEFAULT_OPTIONS.merge \
+ :css => :style,
+ :wrap => :div
+
+ end
+
+end
+end
--- /dev/null
+($:.unshift '../..'; require 'coderay') unless defined? CodeRay
+module CodeRay
+module Encoders
+
+ class Filter < Encoder
+
+ register_for :filter
+
+ protected
+ def setup options
+ @out = Tokens.new
+ end
+
+ def text_token text, kind
+ [text, kind] if include_text_token? text, kind
+ end
+
+ def include_text_token? text, kind
+ true
+ end
+
+ def block_token action, kind
+ [action, kind] if include_block_token? action, kind
+ end
+
+ def include_block_token? action, kind
+ true
+ end
+
+ end
+
+end
+end
+
+if $0 == __FILE__
+ $VERBOSE = true
+ $: << File.join(File.dirname(__FILE__), '..')
+ eval DATA.read, nil, $0, __LINE__ + 4
+end
+
+__END__
+require 'test/unit'
+
+class FilterTest < Test::Unit::TestCase
+
+ def test_creation
+ assert CodeRay::Encoders::Filter < CodeRay::Encoders::Encoder
+ filter = nil
+ assert_nothing_raised do
+ filter = CodeRay.encoder :filter
+ end
+ assert_kind_of CodeRay::Encoders::Encoder, filter
+ end
+
+ def test_filtering_text_tokens
+ tokens = CodeRay::Tokens.new
+ 10.times do |i|
+ tokens << [i.to_s, :index]
+ end
+ assert_equal tokens, CodeRay::Encoders::Filter.new.encode_tokens(tokens)
+ assert_equal tokens, tokens.filter
+ end
+
+ def test_filtering_block_tokens
+ tokens = CodeRay::Tokens.new
+ 10.times do |i|
+ tokens << [:open, :index]
+ tokens << [i.to_s, :content]
+ tokens << [:close, :index]
+ end
+ assert_equal tokens, CodeRay::Encoders::Filter.new.encode_tokens(tokens)
+ assert_equal tokens, tokens.filter
+ end
+
+end
--- /dev/null
+require 'set'
+
+module CodeRay
+module Encoders
+
+ # = HTML Encoder
+ #
+ # This is CodeRay's most important highlighter:
+ # It provides save, fast XHTML generation and CSS support.
+ #
+ # == Usage
+ #
+ # require 'coderay'
+ # puts CodeRay.scan('Some /code/', :ruby).html #-> a HTML page
+ # puts CodeRay.scan('Some /code/', :ruby).html(:wrap => :span)
+ # #-> <span class="CodeRay"><span class="co">Some</span> /code/</span>
+ # puts CodeRay.scan('Some /code/', :ruby).span #-> the same
+ #
+ # puts CodeRay.scan('Some code', :ruby).html(
+ # :wrap => nil,
+ # :line_numbers => :inline,
+ # :css => :style
+ # )
+ # #-> <span class="no">1</span> <span style="color:#036; font-weight:bold;">Some</span> code
+ #
+ # == Options
+ #
+ # === :tab_width
+ # Convert \t characters to +n+ spaces (a number.)
+ # Default: 8
+ #
+ # === :css
+ # How to include the styles; can be :class or :style.
+ #
+ # Default: :class
+ #
+ # === :wrap
+ # Wrap in :page, :div, :span or nil.
+ #
+ # You can also use Encoders::Div and Encoders::Span.
+ #
+ # Default: nil
+ #
+ # === :title
+ #
+ # The title of the HTML page (works only when :wrap is set to :page.)
+ #
+ # Default: 'CodeRay output'
+ #
+ # === :line_numbers
+ # Include line numbers in :table, :inline, :list or nil (no line numbers)
+ #
+ # Default: nil
+ #
+ # === :line_number_start
+ # Where to start with line number counting.
+ #
+ # Default: 1
+ #
+ # === :bold_every
+ # Make every +n+-th number appear bold.
+ #
+ # Default: 10
+ #
+ # === :highlight_lines
+ #
+ # Highlights certain line numbers.
+ # Can be any Enumerable, typically just an Array or Range, of numbers.
+ #
+ # Bolding is deactivated when :highlight_lines is set. It only makes sense
+ # in combination with :line_numbers.
+ #
+ # Default: nil
+ #
+ # === :hint
+ # Include some information into the output using the title attribute.
+ # Can be :info (show token type on mouse-over), :info_long (with full path)
+ # or :debug (via inspect).
+ #
+ # Default: false
+ class HTML < Encoder
+
+ include Streamable
+ register_for :html
+
+ FILE_EXTENSION = 'html'
+
+ DEFAULT_OPTIONS = {
+ :tab_width => 8,
+
+ :css => :class,
+
+ :style => :cycnus,
+ :wrap => nil,
+ :title => 'CodeRay output',
+
+ :line_numbers => nil,
+ :line_number_start => 1,
+ :bold_every => 10,
+ :highlight_lines => nil,
+
+ :hint => false,
+ }
+
+ helper :output, :css
+
+ attr_reader :css
+
+ protected
+
+ HTML_ESCAPE = { #:nodoc:
+ '&' => '&',
+ '"' => '"',
+ '>' => '>',
+ '<' => '<',
+ }
+
+ # This was to prevent illegal HTML.
+ # Strange chars should still be avoided in codes.
+ evil_chars = Array(0x00...0x20) - [?\n, ?\t, ?\s]
+ evil_chars.each { |i| HTML_ESCAPE[i.chr] = ' ' }
+ #ansi_chars = Array(0x7f..0xff)
+ #ansi_chars.each { |i| HTML_ESCAPE[i.chr] = '&#%d;' % i }
+ # \x9 (\t) and \xA (\n) not included
+ #HTML_ESCAPE_PATTERN = /[\t&"><\0-\x8\xB-\x1f\x7f-\xff]/
+ HTML_ESCAPE_PATTERN = /[\t"&><\0-\x8\xB-\x1f]/
+
+ TOKEN_KIND_TO_INFO = Hash.new { |h, kind|
+ h[kind] =
+ case kind
+ when :pre_constant
+ 'Predefined constant'
+ else
+ kind.to_s.gsub(/_/, ' ').gsub(/\b\w/) { $&.capitalize }
+ end
+ }
+
+ TRANSPARENT_TOKEN_KINDS = [
+ :delimiter, :modifier, :content, :escape, :inline_delimiter,
+ ].to_set
+
+ # Generate a hint about the given +classes+ in a +hint+ style.
+ #
+ # +hint+ may be :info, :info_long or :debug.
+ def self.token_path_to_hint hint, classes
+ title =
+ case hint
+ when :info
+ TOKEN_KIND_TO_INFO[classes.first]
+ when :info_long
+ classes.reverse.map { |kind| TOKEN_KIND_TO_INFO[kind] }.join('/')
+ when :debug
+ classes.inspect
+ end
+ title ? " title=\"#{title}\"" : ''
+ end
+
+ def setup options
+ super
+
+ @HTML_ESCAPE = HTML_ESCAPE.dup
+ @HTML_ESCAPE["\t"] = ' ' * options[:tab_width]
+
+ @opened = [nil]
+ @css = CSS.new options[:style]
+
+ hint = options[:hint]
+ if hint and not [:debug, :info, :info_long].include? hint
+ raise ArgumentError, "Unknown value %p for :hint; \
+ expected :info, :debug, false, or nil." % hint
+ end
+
+ case options[:css]
+
+ when :class
+ @css_style = Hash.new do |h, k|
+ c = CodeRay::Tokens::ClassOfKind[k.first]
+ if c == :NO_HIGHLIGHT and not hint
+ h[k.dup] = false
+ else
+ title = if hint
+ HTML.token_path_to_hint(hint, k[1..-1] << k.first)
+ else
+ ''
+ end
+ if c == :NO_HIGHLIGHT
+ h[k.dup] = '<span%s>' % [title]
+ else
+ h[k.dup] = '<span%s class="%s">' % [title, c]
+ end
+ end
+ end
+
+ when :style
+ @css_style = Hash.new do |h, k|
+ if k.is_a? ::Array
+ styles = k.dup
+ else
+ styles = [k]
+ end
+ type = styles.first
+ classes = styles.map { |c| Tokens::ClassOfKind[c] }
+ if classes.first == :NO_HIGHLIGHT and not hint
+ h[k] = false
+ else
+ styles.shift if TRANSPARENT_TOKEN_KINDS.include? styles.first
+ title = HTML.token_path_to_hint hint, styles
+ style = @css[*classes]
+ h[k] =
+ if style
+ '<span%s style="%s">' % [title, style]
+ else
+ false
+ end
+ end
+ end
+
+ else
+ raise ArgumentError, "Unknown value %p for :css." % options[:css]
+
+ end
+ end
+
+ def finish options
+ not_needed = @opened.shift
+ @out << '</span>' * @opened.size
+ unless @opened.empty?
+ warn '%d tokens still open: %p' % [@opened.size, @opened]
+ end
+
+ @out.extend Output
+ @out.css = @css
+ @out.numerize! options[:line_numbers], options
+ @out.wrap! options[:wrap]
+ @out.apply_title! options[:title]
+
+ super
+ end
+
+ def token text, type = :plain
+ case text
+
+ when nil
+ # raise 'Token with nil as text was given: %p' % [[text, type]]
+
+ when String
+ if text =~ /#{HTML_ESCAPE_PATTERN}/o
+ text = text.gsub(/#{HTML_ESCAPE_PATTERN}/o) { |m| @HTML_ESCAPE[m] }
+ end
+ @opened[0] = type
+ if text != "\n" && style = @css_style[@opened]
+ @out << style << text << '</span>'
+ else
+ @out << text
+ end
+
+
+ # token groups, eg. strings
+ when :open
+ @opened[0] = type
+ @out << (@css_style[@opened] || '<span>')
+ @opened << type
+ when :close
+ if @opened.empty?
+ # nothing to close
+ else
+ if $CODERAY_DEBUG and (@opened.size == 1 or @opened.last != type)
+ raise 'Malformed token stream: Trying to close a token (%p) \
+ that is not open. Open are: %p.' % [type, @opened[1..-1]]
+ end
+ @out << '</span>'
+ @opened.pop
+ end
+
+ # whole lines to be highlighted, eg. a deleted line in a diff
+ when :begin_line
+ @opened[0] = type
+ if style = @css_style[@opened]
+ if style['class="']
+ @out << style.sub('class="', 'class="line ')
+ else
+ @out << style.sub('>', ' class="line">')
+ end
+ else
+ @out << '<span class="line">'
+ end
+ @opened << type
+ when :end_line
+ if @opened.empty?
+ # nothing to close
+ else
+ if $CODERAY_DEBUG and (@opened.size == 1 or @opened.last != type)
+ raise 'Malformed token stream: Trying to close a line (%p) \
+ that is not open. Open are: %p.' % [type, @opened[1..-1]]
+ end
+ @out << '</span>'
+ @opened.pop
+ end
+
+ else
+ raise 'unknown token kind: %p' % [text]
+
+ end
+ end
+
+ end
+
+end
+end
--- /dev/null
+module CodeRay
+module Encoders
+
+ class HTML
+ class CSS
+
+ attr :stylesheet
+
+ def CSS.load_stylesheet style = nil
+ CodeRay::Styles[style]
+ end
+
+ def initialize style = :default
+ @classes = Hash.new
+ style = CSS.load_stylesheet style
+ @stylesheet = [
+ style::CSS_MAIN_STYLES,
+ style::TOKEN_COLORS.gsub(/^(?!$)/, '.CodeRay ')
+ ].join("\n")
+ parse style::TOKEN_COLORS
+ end
+
+ def [] *styles
+ cl = @classes[styles.first]
+ return '' unless cl
+ style = ''
+ 1.upto(styles.size) do |offset|
+ break if style = cl[styles[offset .. -1]]
+ end
+ # warn 'Style not found: %p' % [styles] if style.empty?
+ return style
+ end
+
+ private
+
+ CSS_CLASS_PATTERN = /
+ ( # $1 = selectors
+ (?:
+ (?: \s* \. [-\w]+ )+
+ \s* ,?
+ )+
+ )
+ \s* \{ \s*
+ ( [^\}]+ )? # $2 = style
+ \s* \} \s*
+ |
+ ( . ) # $3 = error
+ /mx
+ def parse stylesheet
+ stylesheet.scan CSS_CLASS_PATTERN do |selectors, style, error|
+ raise "CSS parse error: '#{error.inspect}' not recognized" if error
+ for selector in selectors.split(',')
+ classes = selector.scan(/[-\w]+/)
+ cl = classes.pop
+ @classes[cl] ||= Hash.new
+ @classes[cl][classes] = style.to_s.strip.delete(' ').chomp(';')
+ end
+ end
+ end
+
+ end
+ end
+
+end
+end
+
+if $0 == __FILE__
+ require 'pp'
+ pp CodeRay::Encoders::HTML::CSS.new
+end
--- /dev/null
+module CodeRay
+module Encoders
+
+ class HTML
+
+ module Output
+
+ def numerize *args
+ clone.numerize!(*args)
+ end
+
+=begin NUMERIZABLE_WRAPPINGS = {
+ :table => [:div, :page, nil],
+ :inline => :all,
+ :list => [:div, :page, nil]
+ }
+ NUMERIZABLE_WRAPPINGS.default = :all
+=end
+ def numerize! mode = :table, options = {}
+ return self unless mode
+
+ options = DEFAULT_OPTIONS.merge options
+
+ start = options[:line_number_start]
+ unless start.is_a? Integer
+ raise ArgumentError, "Invalid value %p for :line_number_start; Integer expected." % start
+ end
+
+ #allowed_wrappings = NUMERIZABLE_WRAPPINGS[mode]
+ #unless allowed_wrappings == :all or allowed_wrappings.include? options[:wrap]
+ # raise ArgumentError, "Can't numerize, :wrap must be in %p, but is %p" % [NUMERIZABLE_WRAPPINGS, options[:wrap]]
+ #end
+
+ bold_every = options[:bold_every]
+ highlight_lines = options[:highlight_lines]
+ bolding =
+ if bold_every == false && highlight_lines == nil
+ proc { |line| line.to_s }
+ elsif highlight_lines.is_a? Enumerable
+ highlight_lines = highlight_lines.to_set
+ proc do |line|
+ if highlight_lines.include? line
+ "<strong class=\"highlighted\">#{line}</strong>" # highlighted line numbers in bold
+ else
+ line.to_s
+ end
+ end
+ elsif bold_every.is_a? Integer
+ raise ArgumentError, ":bolding can't be 0." if bold_every == 0
+ proc do |line|
+ if line % bold_every == 0
+ "<strong>#{line}</strong>" # every bold_every-th number in bold
+ else
+ line.to_s
+ end
+ end
+ else
+ raise ArgumentError, 'Invalid value %p for :bolding; false or Integer expected.' % bold_every
+ end
+
+ case mode
+ when :inline
+ max_width = (start + line_count).to_s.size
+ line_number = start
+ gsub!(/^/) do
+ line_number_text = bolding.call line_number
+ indent = ' ' * (max_width - line_number.to_s.size) # TODO: Optimize (10^x)
+ res = "<span class=\"no\">#{indent}#{line_number_text}</span> "
+ line_number += 1
+ res
+ end
+
+ when :table
+ # This is really ugly.
+ # Because even monospace fonts seem to have different heights when bold,
+ # I make the newline bold, both in the code and the line numbers.
+ # FIXME Still not working perfect for Mr. Internet Exploder
+ line_numbers = (start ... start + line_count).to_a.map(&bolding).join("\n")
+ line_numbers << "\n" # also for Mr. MS Internet Exploder :-/
+ line_numbers.gsub!(/\n/) { "<tt>\n</tt>" }
+
+ line_numbers_table_tpl = TABLE.apply('LINE_NUMBERS', line_numbers)
+ gsub!("</div>\n", '</div>')
+ gsub!("\n", "<tt>\n</tt>")
+ wrap_in! line_numbers_table_tpl
+ @wrapped_in = :div
+
+ when :list
+ opened_tags = []
+ gsub!(/^.*$\n?/) do |line|
+ line.chomp!
+
+ open = opened_tags.join
+ line.scan(%r!<(/)?span[^>]*>?!) do |close,|
+ if close
+ opened_tags.pop
+ else
+ opened_tags << $&
+ end
+ end
+ close = '</span>' * opened_tags.size
+
+ "<li>#{open}#{line}#{close}</li>\n"
+ end
+ chomp!("\n")
+ wrap_in! LIST
+ @wrapped_in = :div
+
+ else
+ raise ArgumentError, 'Unknown value %p for mode: expected one of %p' %
+ [mode, [:table, :list, :inline]]
+ end
+
+ self
+ end
+
+ def line_count
+ line_count = count("\n")
+ position_of_last_newline = rindex(?\n)
+ if position_of_last_newline
+ after_last_newline = self[position_of_last_newline + 1 .. -1]
+ ends_with_newline = after_last_newline[/\A(?:<\/span>)*\z/]
+ line_count += 1 if not ends_with_newline
+ end
+ line_count
+ end
+
+ end
+
+ end
+
+end
+end
--- /dev/null
+module CodeRay
+module Encoders
+
+ class HTML
+
+ # This module is included in the output String from thew HTML Encoder.
+ #
+ # It provides methods like wrap, div, page etc.
+ #
+ # Remember to use #clone instead of #dup to keep the modules the object was
+ # extended with.
+ #
+ # TODO: more doc.
+ module Output
+
+ require 'coderay/encoders/html/numerization.rb'
+
+ attr_accessor :css
+
+ class << self
+
+ # This makes Output look like a class.
+ #
+ # Example:
+ #
+ # a = Output.new '<span class="co">Code</span>'
+ # a.wrap! :page
+ def new string, css = CSS.new, element = nil
+ output = string.clone.extend self
+ output.wrapped_in = element
+ output.css = css
+ output
+ end
+
+ # Raises an exception if an object that doesn't respond to to_str is extended by Output,
+ # to prevent users from misuse. Use Module#remove_method to disable.
+ def extended o
+ warn "The Output module is intended to extend instances of String, not #{o.class}." unless o.respond_to? :to_str
+ end
+
+ def make_stylesheet css, in_tag = false
+ sheet = css.stylesheet
+ sheet = <<-CSS if in_tag
+<style type="text/css">
+#{sheet}
+</style>
+ CSS
+ sheet
+ end
+
+ def page_template_for_css css
+ sheet = make_stylesheet css
+ PAGE.apply 'CSS', sheet
+ end
+
+ # Define a new wrapper. This is meta programming.
+ def wrapper *wrappers
+ wrappers.each do |wrapper|
+ define_method wrapper do |*args|
+ wrap wrapper, *args
+ end
+ define_method "#{wrapper}!".to_sym do |*args|
+ wrap! wrapper, *args
+ end
+ end
+ end
+
+ end
+
+ wrapper :div, :span, :page
+
+ def wrapped_in? element
+ wrapped_in == element
+ end
+
+ def wrapped_in
+ @wrapped_in ||= nil
+ end
+ attr_writer :wrapped_in
+
+ def wrap_in template
+ clone.wrap_in! template
+ end
+
+ def wrap_in! template
+ Template.wrap! self, template, 'CONTENT'
+ self
+ end
+
+ def apply_title! title
+ self.sub!(/(<title>)(<\/title>)/) { $1 + title + $2 }
+ self
+ end
+
+ def wrap! element, *args
+ return self if not element or element == wrapped_in
+ case element
+ when :div
+ raise "Can't wrap %p in %p" % [wrapped_in, element] unless wrapped_in? nil
+ wrap_in! DIV
+ when :span
+ raise "Can't wrap %p in %p" % [wrapped_in, element] unless wrapped_in? nil
+ wrap_in! SPAN
+ when :page
+ wrap! :div if wrapped_in? nil
+ raise "Can't wrap %p in %p" % [wrapped_in, element] unless wrapped_in? :div
+ wrap_in! Output.page_template_for_css(@css)
+ if args.first.is_a?(Hash) && title = args.first[:title]
+ apply_title! title
+ end
+ self
+ when nil
+ return self
+ else
+ raise "Unknown value %p for :wrap" % element
+ end
+ @wrapped_in = element
+ self
+ end
+
+ def wrap *args
+ clone.wrap!(*args)
+ end
+
+ def stylesheet in_tag = false
+ Output.make_stylesheet @css, in_tag
+ end
+
+ class Template < String
+
+ def self.wrap! str, template, target
+ target = Regexp.new(Regexp.escape("<%#{target}%>"))
+ if template =~ target
+ str[0,0] = $`
+ str << $'
+ else
+ raise "Template target <%%%p%%> not found" % target
+ end
+ end
+
+ def apply target, replacement
+ target = Regexp.new(Regexp.escape("<%#{target}%>"))
+ if self =~ target
+ Template.new($` + replacement + $')
+ else
+ raise "Template target <%%%p%%> not found" % target
+ end
+ end
+
+ module Simple
+ def ` str #` <-- for stupid editors
+ Template.new str
+ end
+ end
+ end
+
+ extend Template::Simple
+
+#-- don't include the templates in docu
+
+ SPAN = `<span class="CodeRay"><%CONTENT%></span>`
+
+ DIV = <<-`DIV`
+<div class="CodeRay">
+ <div class="code"><pre><%CONTENT%></pre></div>
+</div>
+ DIV
+
+ TABLE = <<-`TABLE`
+<table class="CodeRay"><tr>
+ <td class="line_numbers" title="click to toggle" onclick="with (this.firstChild.style) { display = (display == '') ? 'none' : '' }"><pre><%LINE_NUMBERS%></pre></td>
+ <td class="code"><pre ondblclick="with (this.style) { overflow = (overflow == 'auto' || overflow == '') ? 'visible' : 'auto' }"><%CONTENT%></pre></td>
+</tr></table>
+ TABLE
+ # title="double click to expand"
+
+ LIST = <<-`LIST`
+<ol class="CodeRay">
+<%CONTENT%>
+</ol>
+ LIST
+
+ PAGE = <<-`PAGE`
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
+ "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="de">
+<head>
+ <meta http-equiv="content-type" content="text/html; charset=utf-8" />
+ <title></title>
+ <style type="text/css">
+<%CSS%>
+ </style>
+</head>
+<body style="background-color: white;">
+
+<%CONTENT%>
+</body>
+</html>
+ PAGE
+
+ end
+
+ end
+
+end
+end
--- /dev/null
+($:.unshift '../..'; require 'coderay') unless defined? CodeRay
+module CodeRay
+module Encoders
+
+ # = JSON Encoder
+ class JSON < Encoder
+
+ register_for :json
+ FILE_EXTENSION = 'json'
+
+ protected
+ def setup options
+ begin
+ require 'json'
+ rescue LoadError
+ require 'rubygems'
+ require 'json'
+ end
+ @out = []
+ end
+
+ def text_token text, kind
+ { :type => 'text', :text => text, :kind => kind }
+ end
+
+ def block_token action, kind
+ { :type => 'block', :action => action, :kind => kind }
+ end
+
+ def finish options
+ @out.to_json
+ end
+
+ end
+
+end
+end
+
+if $0 == __FILE__
+ $VERBOSE = true
+ $: << File.join(File.dirname(__FILE__), '..')
+ eval DATA.read, nil, $0, __LINE__ + 4
+end
+
+__END__
+require 'test/unit'
+$:.delete '.'
+require 'rubygems' if RUBY_VERSION < '1.9'
+
+class JSONEncoderTest < Test::Unit::TestCase
+
+ def test_json_output
+ tokens = CodeRay.scan <<-RUBY, :ruby
+puts "Hello world!"
+ RUBY
+ require 'json'
+ assert_equal [
+ {"type"=>"text", "text"=>"puts", "kind"=>"ident"},
+ {"type"=>"text", "text"=>" ", "kind"=>"space"},
+ {"type"=>"block", "action"=>"open", "kind"=>"string"},
+ {"type"=>"text", "text"=>"\"", "kind"=>"delimiter"},
+ {"type"=>"text", "text"=>"Hello world!", "kind"=>"content"},
+ {"type"=>"text", "text"=>"\"", "kind"=>"delimiter"},
+ {"type"=>"block", "action"=>"close", "kind"=>"string"},
+ {"type"=>"text", "text"=>"\n", "kind"=>"space"}
+ ], JSON.load(tokens.json)
+ end
+
+end
\ No newline at end of file
--- /dev/null
+($:.unshift '../..'; require 'coderay') unless defined? CodeRay
+module CodeRay
+module Encoders
+
+ # Counts the LoC (Lines of Code). Returns an Integer >= 0.
+ #
+ # Alias: :loc
+ #
+ # Everything that is not comment, markup, doctype/shebang, or an empty line,
+ # is considered to be code.
+ #
+ # For example,
+ # * HTML files not containing JavaScript have 0 LoC
+ # * in a Java class without comments, LoC is the number of non-empty lines
+ #
+ # A Scanner class should define the token kinds that are not code in the
+ # KINDS_NOT_LOC constant, which defaults to [:comment, :doctype].
+ class LinesOfCode < Encoder
+
+ register_for :lines_of_code
+
+ NON_EMPTY_LINE = /^\s*\S.*$/
+
+ def compile tokens, options
+ if scanner = tokens.scanner
+ kinds_not_loc = scanner.class::KINDS_NOT_LOC
+ else
+ warn ArgumentError, 'Tokens have no scanner.' if $VERBOSE
+ kinds_not_loc = CodeRay::Scanners::Scanner::KINDS_NOT_LOC
+ end
+ code = tokens.token_class_filter :exclude => kinds_not_loc
+ @loc = code.text.scan(NON_EMPTY_LINE).size
+ end
+
+ def finish options
+ @loc
+ end
+
+ end
+
+end
+end
+
+if $0 == __FILE__
+ $VERBOSE = true
+ $: << File.join(File.dirname(__FILE__), '..')
+ eval DATA.read, nil, $0, __LINE__ + 4
+end
+
+__END__
+require 'test/unit'
+
+class LinesOfCodeTest < Test::Unit::TestCase
+
+ def test_creation
+ assert CodeRay::Encoders::LinesOfCode < CodeRay::Encoders::Encoder
+ filter = nil
+ assert_nothing_raised do
+ filter = CodeRay.encoder :loc
+ end
+ assert_kind_of CodeRay::Encoders::LinesOfCode, filter
+ assert_nothing_raised do
+ filter = CodeRay.encoder :lines_of_code
+ end
+ assert_kind_of CodeRay::Encoders::LinesOfCode, filter
+ end
+
+ def test_lines_of_code
+ tokens = CodeRay.scan <<-RUBY, :ruby
+#!/usr/bin/env ruby
+
+# a minimal Ruby program
+puts "Hello world!"
+ RUBY
+ assert_equal 1, CodeRay::Encoders::LinesOfCode.new.encode_tokens(tokens)
+ assert_equal 1, tokens.lines_of_code
+ assert_equal 1, tokens.loc
+ end
+
+ def test_filtering_block_tokens
+ tokens = CodeRay::Tokens.new
+ tokens << ["Hello\n", :world]
+ tokens << ["Hello\n", :space]
+ tokens << ["Hello\n", :comment]
+ assert_equal 2, CodeRay::Encoders::LinesOfCode.new.encode_tokens(tokens)
+ assert_equal 2, tokens.lines_of_code
+ assert_equal 2, tokens.loc
+ end
+
+end
\ No newline at end of file
--- /dev/null
+module CodeRay
+module Encoders
+
+ # = Null Encoder
+ #
+ # Does nothing and returns an empty string.
+ class Null < Encoder
+
+ include Streamable
+ register_for :null
+
+ # Defined for faster processing
+ def to_proc
+ proc {}
+ end
+
+ protected
+
+ def token(*)
+ # do nothing
+ end
+
+ end
+
+end
+end
--- /dev/null
+module CodeRay
+module Encoders
+
+ load :html
+
+ class Page < HTML
+
+ FILE_EXTENSION = 'html'
+
+ register_for :page
+
+ DEFAULT_OPTIONS = HTML::DEFAULT_OPTIONS.merge \
+ :css => :class,
+ :wrap => :page,
+ :line_numbers => :table
+
+ end
+
+end
+end
--- /dev/null
+module CodeRay
+module Encoders
+
+ load :html
+
+ class Span < HTML
+
+ FILE_EXTENSION = 'span.html'
+
+ register_for :span
+
+ DEFAULT_OPTIONS = HTML::DEFAULT_OPTIONS.merge \
+ :css => :style,
+ :wrap => :span
+
+ end
+
+end
+end
--- /dev/null
+module CodeRay
+module Encoders
+
+ # Makes a statistic for the given tokens.
+ class Statistic < Encoder
+
+ include Streamable
+ register_for :stats, :statistic
+
+ attr_reader :type_stats, :real_token_count
+
+ protected
+
+ TypeStats = Struct.new :count, :size
+
+ def setup options
+ @type_stats = Hash.new { |h, k| h[k] = TypeStats.new 0, 0 }
+ @real_token_count = 0
+ end
+
+ def generate tokens, options
+ @tokens = tokens
+ super
+ end
+
+ def text_token text, kind
+ @real_token_count += 1 unless kind == :space
+ @type_stats[kind].count += 1
+ @type_stats[kind].size += text.size
+ @type_stats['TOTAL'].size += text.size
+ @type_stats['TOTAL'].count += 1
+ end
+
+ # TODO Hierarchy handling
+ def block_token action, kind
+ @type_stats['TOTAL'].count += 1
+ @type_stats['open/close'].count += 1
+ end
+
+ STATS = <<-STATS
+
+Code Statistics
+
+Tokens %8d
+ Non-Whitespace %8d
+Bytes Total %8d
+
+Token Types (%d):
+ type count ratio size (average)
+-------------------------------------------------------------
+%s
+ STATS
+# space 12007 33.81 % 1.7
+ TOKEN_TYPES_ROW = <<-TKR
+ %-20s %8d %6.2f %% %5.1f
+ TKR
+
+ def finish options
+ all = @type_stats['TOTAL']
+ all_count, all_size = all.count, all.size
+ @type_stats.each do |type, stat|
+ stat.size /= stat.count.to_f
+ end
+ types_stats = @type_stats.sort_by { |k, v| [-v.count, k.to_s] }.map do |k, v|
+ TOKEN_TYPES_ROW % [k, v.count, 100.0 * v.count / all_count, v.size]
+ end.join
+ STATS % [
+ all_count, @real_token_count, all_size,
+ @type_stats.delete_if { |k, v| k.is_a? String }.size,
+ types_stats
+ ]
+ end
+
+ end
+
+end
+end
--- /dev/null
+# encoders/term.rb
+# By Rob Aldred (http://robaldred.co.uk)
+# Based on idea by Nathan Weizenbaum (http://nex-3.com)
+# MIT License (http://www.opensource.org/licenses/mit-license.php)
+#
+# A CodeRay encoder that outputs code highlighted for a color terminal.
+# Check out http://robaldred.co.uk
+
+module CodeRay
+ module Encoders
+ class Term < Encoder
+ register_for :term
+
+ TOKEN_COLORS = {
+ :annotation => '35',
+ :attribute_name => '33',
+ :attribute_name_fat => '33',
+ :attribute_value => '31',
+ :attribute_value_fat => '31',
+ :bin => '1;35',
+ :char => {:self => '36', :delimiter => '34'},
+ :class => '1;35',
+ :class_variable => '36',
+ :color => '32',
+ :comment => '37',
+ :complex => '34',
+ :constant => ['34', '4'],
+ :decoration => '35',
+ :definition => '1;32',
+ :directive => ['32', '4'],
+ :doc => '46',
+ :doctype => '1;30',
+ :doc_string => ['31', '4'],
+ :entity => '33',
+ :error => ['1;33', '41'],
+ :exception => '1;31',
+ :float => '1;35',
+ :function => '1;34',
+ :global_variable => '42',
+ :hex => '1;36',
+ :important => '1;31',
+ :include => '33',
+ :integer => '1;34',
+ :interpreted => '1;35',
+ :key => '35',
+ :label => '1;4',
+ :local_variable => '33',
+ :oct => '1;35',
+ :operator_name => '1;29',
+ :pre_constant => '1;36',
+ :pre_type => '1;30',
+ :predefined => ['4', '1;34'],
+ :preprocessor => '36',
+ :pseudo_class => '34',
+ :regexp => {
+ :content => '31',
+ :delimiter => '1;29',
+ :modifier => '35',
+ :function => '1;29'
+ },
+ :reserved => '1;31',
+ :shell => {
+ :self => '42',
+ :content => '1;29',
+ :delimiter => '37',
+ },
+ :string => {
+ :self => '32',
+ :modifier => '1;32',
+ :escape => '1;36',
+ :delimiter => '1;32',
+ },
+ :symbol => '1;32',
+ :tag => '34',
+ :tag_fat => '1;34',
+ :tag_special => ['34', '4'],
+ :type => '1;34',
+ :value => '36',
+ :variable => '34',
+ :insert => '42',
+ :delete => '41',
+ :change => '44',
+ :head => '45',
+ }
+ TOKEN_COLORS[:keyword] = TOKEN_COLORS[:reserved]
+ TOKEN_COLORS[:method] = TOKEN_COLORS[:function]
+ TOKEN_COLORS[:imaginary] = TOKEN_COLORS[:complex]
+ TOKEN_COLORS[:open] = TOKEN_COLORS[:close] = TOKEN_COLORS[:nesting_delimiter] = TOKEN_COLORS[:escape] = TOKEN_COLORS[:delimiter]
+
+ protected
+
+ def setup(options)
+ @out = ''
+ @opened = [nil]
+ @subcolors = nil
+ end
+
+ def finish(options)
+ super
+ end
+
+ def token text, type = :plain
+ case text
+
+ when nil
+ # raise 'Token with nil as text was given: %p' % [[text, type]]
+
+ when String
+
+ if color = (@subcolors || TOKEN_COLORS)[type]
+ color = color[:self] || return if Hash === color
+
+ @out << col(color) + text.gsub("\n", col(0) + "\n" + col(color)) + col(0)
+ @out << col(@subcolors[:self]) if @subcolors && @subcolors[:self]
+ else
+ @out << text
+ end
+
+ # token groups, eg. strings
+ when :open
+ @opened[0] = type
+ if color = TOKEN_COLORS[type]
+ if Hash === color
+ @subcolors = color
+ @out << col(color[:self]) if color[:self]
+ else
+ @subcolors = {}
+ @out << col(color)
+ end
+ end
+ @opened << type
+ when :close
+ if @opened.empty?
+ # nothing to close
+ else
+ @out << col(0) if (@subcolors || {})[:self]
+ @subcolors = nil
+ @opened.pop
+ end
+
+ # whole lines to be highlighted, eg. a added/modified/deleted lines in a diff
+ when :begin_line
+
+ when :end_line
+
+ else
+ raise 'unknown token kind: %p' % [text]
+ end
+ end
+
+ private
+
+ def col(color)
+ Array(color).map { |c| "\e[#{c}m" }.join
+ end
+ end
+ end
+end
\ No newline at end of file
--- /dev/null
+module CodeRay
+module Encoders
+
+ class Text < Encoder
+
+ include Streamable
+ register_for :text
+
+ FILE_EXTENSION = 'txt'
+
+ DEFAULT_OPTIONS = {
+ :separator => ''
+ }
+
+ protected
+ def setup options
+ super
+ @sep = options[:separator]
+ end
+
+ def text_token text, kind
+ text + @sep
+ end
+
+ def finish options
+ super.chomp @sep
+ end
+
+ end
+
+end
+end
--- /dev/null
+($:.unshift '../..'; require 'coderay') unless defined? CodeRay
+module CodeRay
+module Encoders
+
+ load :filter
+
+ class TokenClassFilter < Filter
+
+ include Streamable
+ register_for :token_class_filter
+
+ DEFAULT_OPTIONS = {
+ :exclude => [],
+ :include => :all
+ }
+
+ protected
+ def setup options
+ super
+ @exclude = options[:exclude]
+ @exclude = Array(@exclude) unless @exclude == :all
+ @include = options[:include]
+ @include = Array(@include) unless @include == :all
+ end
+
+ def include_text_token? text, kind
+ (@include == :all || @include.include?(kind)) &&
+ !(@exclude == :all || @exclude.include?(kind))
+ end
+
+ end
+
+end
+end
+
+if $0 == __FILE__
+ $VERBOSE = true
+ $: << File.join(File.dirname(__FILE__), '..')
+ eval DATA.read, nil, $0, __LINE__ + 4
+end
+
+__END__
+require 'test/unit'
+
+class TokenClassFilterTest < Test::Unit::TestCase
+
+ def test_creation
+ assert CodeRay::Encoders::TokenClassFilter < CodeRay::Encoders::Encoder
+ assert CodeRay::Encoders::TokenClassFilter < CodeRay::Encoders::Filter
+ filter = nil
+ assert_nothing_raised do
+ filter = CodeRay.encoder :token_class_filter
+ end
+ assert_instance_of CodeRay::Encoders::TokenClassFilter, filter
+ end
+
+ def test_filtering_text_tokens
+ tokens = CodeRay::Tokens.new
+ for i in 1..10
+ tokens << [i.to_s, :index]
+ tokens << [' ', :space] if i < 10
+ end
+ assert_equal 10, CodeRay::Encoders::TokenClassFilter.new.encode_tokens(tokens, :exclude => :space).size
+ assert_equal 10, tokens.token_class_filter(:exclude => :space).size
+ assert_equal 9, CodeRay::Encoders::TokenClassFilter.new.encode_tokens(tokens, :include => :space).size
+ assert_equal 9, tokens.token_class_filter(:include => :space).size
+ assert_equal 0, CodeRay::Encoders::TokenClassFilter.new.encode_tokens(tokens, :exclude => :all).size
+ assert_equal 0, tokens.token_class_filter(:exclude => :all).size
+ end
+
+ def test_filtering_block_tokens
+ tokens = CodeRay::Tokens.new
+ 10.times do |i|
+ tokens << [:open, :index]
+ tokens << [i.to_s, :content]
+ tokens << [:close, :index]
+ end
+ assert_equal 20, CodeRay::Encoders::TokenClassFilter.new.encode_tokens(tokens, :include => :blubb).size
+ assert_equal 20, tokens.token_class_filter(:include => :blubb).size
+ assert_equal 30, CodeRay::Encoders::TokenClassFilter.new.encode_tokens(tokens, :exclude => :index).size
+ assert_equal 30, tokens.token_class_filter(:exclude => :index).size
+ end
+
+end
--- /dev/null
+module CodeRay
+module Encoders
+
+ # = XML Encoder
+ #
+ # Uses REXML. Very slow.
+ class XML < Encoder
+
+ include Streamable
+ register_for :xml
+
+ FILE_EXTENSION = 'xml'
+
+ require 'rexml/document'
+
+ DEFAULT_OPTIONS = {
+ :tab_width => 8,
+ :pretty => -1,
+ :transitive => false,
+ }
+
+ protected
+
+ def setup options
+ @doc = REXML::Document.new
+ @doc << REXML::XMLDecl.new
+ @tab_width = options[:tab_width]
+ @root = @node = @doc.add_element('coderay-tokens')
+ end
+
+ def finish options
+ @out = ''
+ @doc.write @out, options[:pretty], options[:transitive], true
+ @out
+ end
+
+ def text_token text, kind
+ if kind == :space
+ token = @node
+ else
+ token = @node.add_element kind.to_s
+ end
+ text.scan(/(\x20+)|(\t+)|(\n)|[^\x20\t\n]+/) do |space, tab, nl|
+ case
+ when space
+ token << REXML::Text.new(space, true)
+ when tab
+ token << REXML::Text.new(tab, true)
+ when nl
+ token << REXML::Text.new(nl, true)
+ else
+ token << REXML::Text.new($&)
+ end
+ end
+ end
+
+ def open_token kind
+ @node = @node.add_element kind.to_s
+ end
+
+ def close_token kind
+ if @node == @root
+ raise 'no token to close!'
+ end
+ @node = @node.parent
+ end
+
+ end
+
+end
+end
--- /dev/null
+module CodeRay
+module Encoders
+
+ # = YAML Encoder
+ #
+ # Slow.
+ class YAML < Encoder
+
+ register_for :yaml
+
+ FILE_EXTENSION = 'yaml'
+
+ protected
+ def compile tokens, options
+ require 'yaml'
+ @out = tokens.to_a.to_yaml
+ end
+
+ end
+
+end
+end
--- /dev/null
+module CodeRay
+
+ # A little hack to enable CodeRay highlighting in RedCloth.
+ #
+ # Usage:
+ # require 'coderay'
+ # require 'coderay/for_redcloth'
+ # RedCloth.new('@[ruby]puts "Hello, World!"@').to_html
+ #
+ # Make sure you have RedCloth 4.0.3 activated, for example by calling
+ # require 'rubygems'
+ # before RedCloth is loaded and before calling CodeRay.for_redcloth.
+ module ForRedCloth
+
+ def self.install
+ gem 'RedCloth', '>= 4.0.3' if defined? gem
+ require 'redcloth'
+ unless RedCloth::VERSION.to_s >= '4.0.3'
+ if defined? gem
+ raise 'CodeRay.for_redcloth needs RedCloth version 4.0.3 or later. ' +
+ "You have #{RedCloth::VERSION}. Please gem install RedCloth."
+ else
+ $".delete 'redcloth.rb' # sorry, but it works
+ require 'rubygems'
+ return install # retry
+ end
+ end
+ unless RedCloth::VERSION.to_s >= '4.2.2'
+ warn 'CodeRay.for_redcloth works best with RedCloth version 4.2.2 or later.'
+ end
+ RedCloth::TextileDoc.send :include, ForRedCloth::TextileDoc
+ RedCloth::Formatters::HTML.module_eval do
+ def unescape(html)
+ replacements = {
+ '&' => '&',
+ '"' => '"',
+ '>' => '>',
+ '<' => '<',
+ }
+ html.gsub(/&(?:amp|quot|[gl]t);/) { |entity| replacements[entity] }
+ end
+ undef code, bc_open, bc_close, escape_pre
+ def code(opts) # :nodoc:
+ opts[:block] = true
+ if !opts[:lang] && RedCloth::VERSION.to_s >= '4.2.0'
+ # simulating pre-4.2 behavior
+ if opts[:text].sub!(/\A\[(\w+)\]/, '')
+ if CodeRay::Scanners[$1].plugin_id == 'plaintext'
+ opts[:text] = $& + opts[:text]
+ else
+ opts[:lang] = $1
+ end
+ end
+ end
+ if opts[:lang] && !filter_coderay
+ require 'coderay'
+ @in_bc ||= nil
+ format = @in_bc ? :div : :span
+ opts[:text] = unescape(opts[:text]) unless @in_bc
+ highlighted_code = CodeRay.encode opts[:text], opts[:lang], format, :stream => true
+ highlighted_code.sub!(/\A<(span|div)/) { |m| m + pba(@in_bc || opts) }
+ highlighted_code
+ else
+ "<code#{pba(opts)}>#{opts[:text]}</code>"
+ end
+ end
+ def bc_open(opts) # :nodoc:
+ opts[:block] = true
+ @in_bc = opts
+ opts[:lang] ? '' : "<pre#{pba(opts)}>"
+ end
+ def bc_close(opts) # :nodoc:
+ opts = @in_bc
+ @in_bc = nil
+ opts[:lang] ? '' : "</pre>\n"
+ end
+ def escape_pre(text)
+ if @in_bc ||= nil
+ text
+ else
+ html_esc(text, :html_escape_preformatted)
+ end
+ end
+ end
+ end
+
+ module TextileDoc # :nodoc:
+ attr_accessor :filter_coderay
+ end
+
+ end
+
+end
+
+CodeRay::ForRedCloth.install
\ No newline at end of file
--- /dev/null
+#!/usr/bin/env ruby
+module CodeRay
+
+# = FileType
+#
+# A simple filetype recognizer.
+#
+# Copyright (c) 2006 by murphy (Kornelius Kalnbach) <murphy rubychan de>
+#
+# License:: LGPL / ask the author
+# Version:: 0.1 (2005-09-01)
+#
+# == Documentation
+#
+# # determine the type of the given
+# lang = FileType[ARGV.first]
+#
+# # return :plaintext if the file type is unknown
+# lang = FileType.fetch ARGV.first, :plaintext
+#
+# # try the shebang line, too
+# lang = FileType.fetch ARGV.first, :plaintext, true
+module FileType
+
+ UnknownFileType = Class.new Exception
+
+ class << self
+
+ # Try to determine the file type of the file.
+ #
+ # +filename+ is a relative or absolute path to a file.
+ #
+ # The file itself is only accessed when +read_shebang+ is set to true.
+ # That means you can get filetypes from files that don't exist.
+ def [] filename, read_shebang = false
+ name = File.basename filename
+ ext = File.extname(name).sub(/^\./, '') # from last dot, delete the leading dot
+ ext2 = filename.to_s[/\.(.*)/, 1] # from first dot
+
+ type =
+ TypeFromExt[ext] ||
+ TypeFromExt[ext.downcase] ||
+ (TypeFromExt[ext2] if ext2) ||
+ (TypeFromExt[ext2.downcase] if ext2) ||
+ TypeFromName[name] ||
+ TypeFromName[name.downcase]
+ type ||= shebang(filename) if read_shebang
+
+ type
+ end
+
+ def shebang filename
+ begin
+ File.open filename, 'r' do |f|
+ if first_line = f.gets
+ if type = first_line[TypeFromShebang]
+ type.to_sym
+ end
+ end
+ end
+ rescue IOError
+ nil
+ end
+ end
+
+ # This works like Hash#fetch.
+ #
+ # If the filetype cannot be found, the +default+ value
+ # is returned.
+ def fetch filename, default = nil, read_shebang = false
+ if default and block_given?
+ warn 'block supersedes default value argument'
+ end
+
+ unless type = self[filename, read_shebang]
+ return yield if block_given?
+ return default if default
+ raise UnknownFileType, 'Could not determine type of %p.' % filename
+ end
+ type
+ end
+
+ end
+
+ TypeFromExt = {
+ 'c' => :c,
+ 'css' => :css,
+ 'diff' => :diff,
+ 'dpr' => :delphi,
+ 'groovy' => :groovy,
+ 'gvy' => :groovy,
+ 'h' => :c,
+ 'htm' => :html,
+ 'html' => :html,
+ 'html.erb' => :rhtml,
+ 'java' => :java,
+ 'js' => :java_script,
+ 'json' => :json,
+ 'mab' => :ruby,
+ 'pas' => :delphi,
+ 'patch' => :diff,
+ 'php' => :php,
+ 'php3' => :php,
+ 'php4' => :php,
+ 'php5' => :php,
+ 'py' => :python,
+ 'py3' => :python,
+ 'pyw' => :python,
+ 'rake' => :ruby,
+ 'raydebug' => :debug,
+ 'rb' => :ruby,
+ 'rbw' => :ruby,
+ 'rhtml' => :rhtml,
+ 'rxml' => :ruby,
+ 'sch' => :scheme,
+ 'sql' => :sql,
+ 'ss' => :scheme,
+ 'xhtml' => :xhtml,
+ 'xml' => :xml,
+ 'yaml' => :yaml,
+ 'yml' => :yaml,
+ }
+ for cpp_alias in %w[cc cpp cp cxx c++ C hh hpp h++ cu]
+ TypeFromExt[cpp_alias] = :cpp
+ end
+
+ TypeFromShebang = /\b(?:ruby|perl|python|sh)\b/
+
+ TypeFromName = {
+ 'Rakefile' => :ruby,
+ 'Rantfile' => :ruby,
+ }
+
+end
+
+end
+
+if $0 == __FILE__
+ $VERBOSE = true
+ eval DATA.read, nil, $0, __LINE__ + 4
+end
+
+__END__
+require 'test/unit'
+
+class FileTypeTests < Test::Unit::TestCase
+
+ include CodeRay
+
+ def test_fetch
+ assert_raise FileType::UnknownFileType do
+ FileType.fetch ''
+ end
+
+ assert_throws :not_found do
+ FileType.fetch '.' do
+ throw :not_found
+ end
+ end
+
+ assert_equal :default, FileType.fetch('c', :default)
+
+ stderr, fake_stderr = $stderr, Object.new
+ $err = ''
+ def fake_stderr.write x
+ $err << x
+ end
+ $stderr = fake_stderr
+ FileType.fetch('c', :default) { }
+ assert_equal "block supersedes default value argument\n", $err
+ $stderr = stderr
+ end
+
+ def test_ruby
+ assert_equal :ruby, FileType['test.rb']
+ assert_equal :ruby, FileType['test.java.rb']
+ assert_equal :java, FileType['test.rb.java']
+ assert_equal :ruby, FileType['C:\\Program Files\\x\\y\\c\\test.rbw']
+ assert_equal :ruby, FileType['/usr/bin/something/Rakefile']
+ assert_equal :ruby, FileType['~/myapp/gem/Rantfile']
+ assert_equal :ruby, FileType['./lib/tasks\repository.rake']
+ assert_not_equal :ruby, FileType['test_rb']
+ assert_not_equal :ruby, FileType['Makefile']
+ assert_not_equal :ruby, FileType['set.rb/set']
+ assert_not_equal :ruby, FileType['~/projects/blabla/rb']
+ end
+
+ def test_c
+ assert_equal :c, FileType['test.c']
+ assert_equal :c, FileType['C:\\Program Files\\x\\y\\c\\test.h']
+ assert_not_equal :c, FileType['test_c']
+ assert_not_equal :c, FileType['Makefile']
+ assert_not_equal :c, FileType['set.h/set']
+ assert_not_equal :c, FileType['~/projects/blabla/c']
+ end
+
+ def test_cpp
+ assert_equal :cpp, FileType['test.c++']
+ assert_equal :cpp, FileType['test.cxx']
+ assert_equal :cpp, FileType['test.hh']
+ assert_equal :cpp, FileType['test.hpp']
+ assert_equal :cpp, FileType['test.cu']
+ assert_equal :cpp, FileType['test.C']
+ assert_not_equal :cpp, FileType['test.c']
+ assert_not_equal :cpp, FileType['test.h']
+ end
+
+ def test_html
+ assert_equal :html, FileType['test.htm']
+ assert_equal :xhtml, FileType['test.xhtml']
+ assert_equal :xhtml, FileType['test.html.xhtml']
+ assert_equal :rhtml, FileType['_form.rhtml']
+ assert_equal :rhtml, FileType['_form.html.erb']
+ end
+
+ def test_yaml
+ assert_equal :yaml, FileType['test.yml']
+ assert_equal :yaml, FileType['test.yaml']
+ assert_equal :yaml, FileType['my.html.yaml']
+ assert_not_equal :yaml, FileType['YAML']
+ end
+
+ def test_pathname
+ require 'pathname'
+ pn = Pathname.new 'test.rb'
+ assert_equal :ruby, FileType[pn]
+ dir = Pathname.new '/etc/var/blubb'
+ assert_equal :ruby, FileType[dir + pn]
+ assert_equal :cpp, FileType[dir + 'test.cpp']
+ end
+
+ def test_no_shebang
+ dir = './test'
+ if File.directory? dir
+ Dir.chdir dir do
+ assert_equal :c, FileType['test.c']
+ end
+ end
+ end
+
+ def test_shebang_empty_file
+ require 'tmpdir'
+ tmpfile = File.join(Dir.tmpdir, 'bla')
+ File.open(tmpfile, 'w') { } # touch
+ assert_equal nil, FileType[tmpfile]
+ end
+
+ def test_shebang
+ require 'tmpdir'
+ tmpfile = File.join(Dir.tmpdir, 'bla')
+ File.open(tmpfile, 'w') { |f| f.puts '#!/usr/bin/env ruby' }
+ assert_equal :ruby, FileType[tmpfile, true]
+ end
+
+end
--- /dev/null
+# =GZip Simple
+#
+# A simplified interface to the gzip library +zlib+ (from the Ruby Standard Library.)
+#
+# Author: murphy (mail to murphy rubychan de)
+#
+# Version: 0.2 (2005.may.28)
+#
+# ==Documentation
+#
+# See +GZip+ module and the +String+ extensions.
+#
+module GZip
+
+ require 'zlib'
+
+ # The default zipping level. 7 zips good and fast.
+ DEFAULT_GZIP_LEVEL = 7
+
+ # Unzips the given string +s+.
+ #
+ # Example:
+ # require 'gzip_simple'
+ # print GZip.gunzip(File.read('adresses.gz'))
+ def GZip.gunzip s
+ Zlib::Inflate.inflate s
+ end
+
+ # Zips the given string +s+.
+ #
+ # Example:
+ # require 'gzip_simple'
+ # File.open('adresses.gz', 'w') do |file
+ # file.write GZip.gzip('Mum: 0123 456 789', 9)
+ # end
+ #
+ # If you provide a +level+, you can control how strong
+ # the string is compressed:
+ # - 0: no compression, only convert to gzip format
+ # - 1: compress fast
+ # - 7: compress more, but still fast (default)
+ # - 8: compress more, slower
+ # - 9: compress best, very slow
+ def GZip.gzip s, level = DEFAULT_GZIP_LEVEL
+ Zlib::Deflate.new(level).deflate s, Zlib::FINISH
+ end
+end
+
+
+# String extensions to use the GZip module.
+#
+# The methods gzip and gunzip provide an even more simple
+# interface to the ZLib:
+#
+# # create a big string
+# x = 'a' * 1000
+#
+# # zip it
+# x_gz = x.gzip
+#
+# # test the result
+# puts 'Zipped %d bytes to %d bytes.' % [x.size, x_gz.size]
+# #-> Zipped 1000 bytes to 19 bytes.
+#
+# # unzipping works
+# p x_gz.gunzip == x #-> true
+class String
+ # Returns the string, unzipped.
+ # See GZip.gunzip
+ def gunzip
+ GZip.gunzip self
+ end
+ # Replaces the string with its unzipped value.
+ # See GZip.gunzip
+ def gunzip!
+ replace gunzip
+ end
+
+ # Returns the string, zipped.
+ # +level+ is the gzip compression level, see GZip.gzip.
+ def gzip level = GZip::DEFAULT_GZIP_LEVEL
+ GZip.gzip self, level
+ end
+ # Replaces the string with its zipped value.
+ # See GZip.gzip.
+ def gzip!(*args)
+ replace gzip(*args)
+ end
+end
+
+if $0 == __FILE__
+ eval DATA.read, nil, $0, __LINE__+4
+end
+
+__END__
+#CODE
+
+# Testing / Benchmark
+x = 'a' * 1000
+x_gz = x.gzip
+puts 'Zipped %d bytes to %d bytes.' % [x.size, x_gz.size] #-> Zipped 1000 bytes to 19 bytes.
+p x_gz.gunzip == x #-> true
+
+require 'benchmark'
+
+INFO = 'packed to %0.3f%%' # :nodoc:
+
+x = Array.new(100000) { rand(255).chr + 'aaaaaaaaa' + rand(255).chr }.join
+Benchmark.bm(10) do |bm|
+ for level in 0..9
+ bm.report "zip #{level}" do
+ $x = x.gzip level
+ end
+ puts INFO % [100.0 * $x.size / x.size]
+ end
+ bm.report 'zip' do
+ $x = x.gzip
+ end
+ puts INFO % [100.0 * $x.size / x.size]
+ bm.report 'unzip' do
+ $x.gunzip
+ end
+end
--- /dev/null
+module CodeRay
+
+# = PluginHost
+#
+# A simple subclass plugin system.
+#
+# Example:
+# class Generators < PluginHost
+# plugin_path 'app/generators'
+# end
+#
+# class Generator
+# extend Plugin
+# PLUGIN_HOST = Generators
+# end
+#
+# class FancyGenerator < Generator
+# register_for :fancy
+# end
+#
+# Generators[:fancy] #-> FancyGenerator
+# # or
+# CodeRay.require_plugin 'Generators/fancy'
+module PluginHost
+
+ # Raised if Encoders::[] fails because:
+ # * a file could not be found
+ # * the requested Encoder is not registered
+ PluginNotFound = Class.new Exception
+ HostNotFound = Class.new Exception
+
+ PLUGIN_HOSTS = []
+ PLUGIN_HOSTS_BY_ID = {} # dummy hash
+
+ # Loads all plugins using list and load.
+ def load_all
+ for plugin in list
+ load plugin
+ end
+ end
+
+ # Returns the Plugin for +id+.
+ #
+ # Example:
+ # yaml_plugin = MyPluginHost[:yaml]
+ def [] id, *args, &blk
+ plugin = validate_id(id)
+ begin
+ plugin = plugin_hash.[] plugin, *args, &blk
+ end while plugin.is_a? Symbol
+ plugin
+ end
+
+ # Alias for +[]+.
+ alias load []
+
+ def require_helper plugin_id, helper_name
+ path = path_to File.join(plugin_id, helper_name)
+ require path
+ end
+
+ class << self
+
+ # Adds the module/class to the PLUGIN_HOSTS list.
+ def extended mod
+ PLUGIN_HOSTS << mod
+ end
+
+ # Warns you that you should not #include this module.
+ def included mod
+ warn "#{name} should not be included. Use extend."
+ end
+
+ # Find the PluginHost for host_id.
+ def host_by_id host_id
+ unless PLUGIN_HOSTS_BY_ID.default_proc
+ ph = Hash.new do |h, a_host_id|
+ for host in PLUGIN_HOSTS
+ h[host.host_id] = host
+ end
+ h.fetch a_host_id, nil
+ end
+ PLUGIN_HOSTS_BY_ID.replace ph
+ end
+ PLUGIN_HOSTS_BY_ID[host_id]
+ end
+
+ end
+
+ # The path where the plugins can be found.
+ def plugin_path *args
+ unless args.empty?
+ @plugin_path = File.expand_path File.join(*args)
+ load_map
+ end
+ @plugin_path
+ end
+
+ # The host's ID.
+ #
+ # If PLUGIN_HOST_ID is not set, it is simply the class name.
+ def host_id
+ if self.const_defined? :PLUGIN_HOST_ID
+ self::PLUGIN_HOST_ID
+ else
+ name
+ end
+ end
+
+ # Map a plugin_id to another.
+ #
+ # Usage: Put this in a file plugin_path/_map.rb.
+ #
+ # class MyColorHost < PluginHost
+ # map :navy => :dark_blue,
+ # :maroon => :brown,
+ # :luna => :moon
+ # end
+ def map hash
+ for from, to in hash
+ from = validate_id from
+ to = validate_id to
+ plugin_hash[from] = to unless plugin_hash.has_key? from
+ end
+ end
+
+ # Define the default plugin to use when no plugin is found
+ # for a given id.
+ #
+ # See also map.
+ #
+ # class MyColorHost < PluginHost
+ # map :navy => :dark_blue
+ # default :gray
+ # end
+ def default id = nil
+ if id
+ id = validate_id id
+ plugin_hash[nil] = id
+ else
+ plugin_hash[nil]
+ end
+ end
+
+ # Every plugin must register itself for one or more
+ # +ids+ by calling register_for, which calls this method.
+ #
+ # See Plugin#register_for.
+ def register plugin, *ids
+ for id in ids
+ unless id.is_a? Symbol
+ raise ArgumentError,
+ "id must be a Symbol, but it was a #{id.class}"
+ end
+ plugin_hash[validate_id(id)] = plugin
+ end
+ end
+
+ # A Hash of plugion_id => Plugin pairs.
+ def plugin_hash
+ @plugin_hash ||= create_plugin_hash
+ end
+
+ # Returns an array of all .rb files in the plugin path.
+ #
+ # The extension .rb is not included.
+ def list
+ Dir[path_to('*')].select do |file|
+ File.basename(file)[/^(?!_)\w+\.rb$/]
+ end.map do |file|
+ File.basename file, '.rb'
+ end
+ end
+
+ # Makes a map of all loaded plugins.
+ def inspect
+ map = plugin_hash.dup
+ map.each do |id, plugin|
+ map[id] = plugin.to_s[/(?>\w+)$/]
+ end
+ "#{name}[#{host_id}]#{map.inspect}"
+ end
+
+protected
+ # Created a new plugin list and stores it to @plugin_hash.
+ def create_plugin_hash
+ @plugin_hash =
+ Hash.new do |h, plugin_id|
+ id = validate_id(plugin_id)
+ path = path_to id
+ begin
+ require path
+ rescue LoadError => boom
+ if h.has_key? nil # default plugin
+ h[id] = h[nil]
+ else
+ raise PluginNotFound, 'Could not load plugin %p: %s' % [id, boom]
+ end
+ else
+ # Plugin should have registered by now
+ unless h.has_key? id
+ raise PluginNotFound,
+ "No #{self.name} plugin for #{id.inspect} found in #{path}."
+ end
+ end
+ h[id]
+ end
+ end
+
+ # Loads the map file (see map).
+ #
+ # This is done automatically when plugin_path is called.
+ def load_map
+ mapfile = path_to '_map'
+ if File.exist? mapfile
+ require mapfile
+ elsif $VERBOSE
+ warn 'no _map.rb found for %s' % name
+ end
+ end
+
+ # Returns the Plugin for +id+.
+ # Use it like Hash#fetch.
+ #
+ # Example:
+ # yaml_plugin = MyPluginHost[:yaml, :default]
+ def fetch id, *args, &blk
+ plugin_hash.fetch validate_id(id), *args, &blk
+ end
+
+ # Returns the expected path to the plugin file for the given id.
+ def path_to plugin_id
+ File.join plugin_path, "#{plugin_id}.rb"
+ end
+
+ # Converts +id+ to a Symbol if it is a String,
+ # or returns +id+ if it already is a Symbol.
+ #
+ # Raises +ArgumentError+ for all other objects, or if the
+ # given String includes non-alphanumeric characters (\W).
+ def validate_id id
+ if id.is_a? Symbol or id.nil?
+ id
+ elsif id.is_a? String
+ if id[/\w+/] == id
+ id.downcase.to_sym
+ else
+ raise ArgumentError, "Invalid id: '#{id}' given."
+ end
+ else
+ raise ArgumentError,
+ "String or Symbol expected, but #{id.class} given."
+ end
+ end
+
+end
+
+
+# = Plugin
+#
+# Plugins have to include this module.
+#
+# IMPORTANT: use extend for this module.
+#
+# Example: see PluginHost.
+module Plugin
+
+ def included mod
+ warn "#{name} should not be included. Use extend."
+ end
+
+ # Register this class for the given langs.
+ # Example:
+ # class MyPlugin < PluginHost::BaseClass
+ # register_for :my_id
+ # ...
+ # end
+ #
+ # See PluginHost.register.
+ def register_for *ids
+ plugin_host.register self, *ids
+ end
+
+ # Returns the title of the plugin, or sets it to the
+ # optional argument +title+.
+ def title title = nil
+ if title
+ @title = title.to_s
+ else
+ @title ||= name[/([^:]+)$/, 1]
+ end
+ end
+
+ # The host for this Plugin class.
+ def plugin_host host = nil
+ if host and not host.is_a? PluginHost
+ raise ArgumentError,
+ "PluginHost expected, but #{host.class} given."
+ end
+ self.const_set :PLUGIN_HOST, host if host
+ self::PLUGIN_HOST
+ end
+
+ # Require some helper files.
+ #
+ # Example:
+ #
+ # class MyPlugin < PluginHost::BaseClass
+ # register_for :my_id
+ # helper :my_helper
+ #
+ # The above example loads the file myplugin/my_helper.rb relative to the
+ # file in which MyPlugin was defined.
+ #
+ # You can also load a helper from a different plugin:
+ #
+ # helper 'other_plugin/helper_name'
+ def helper *helpers
+ for helper in helpers
+ if helper.is_a?(String) && helper[/\//]
+ self::PLUGIN_HOST.require_helper $`, $'
+ else
+ self::PLUGIN_HOST.require_helper plugin_id, helper.to_s
+ end
+ end
+ end
+
+ # Returns the pulgin id used by the engine.
+ def plugin_id
+ name[/\w+$/].downcase
+ end
+
+end
+
+# Convenience method for plugin loading.
+# The syntax used is:
+#
+# CodeRay.require_plugin '<Host ID>/<Plugin ID>'
+#
+# Returns the loaded plugin.
+def self.require_plugin path
+ host_id, plugin_id = path.split '/', 2
+ host = PluginHost.host_by_id(host_id)
+ raise PluginHost::HostNotFound,
+ "No host for #{host_id.inspect} found." unless host
+ host.load plugin_id
+end
+
+end
\ No newline at end of file
--- /dev/null
+module CodeRay
+
+# = WordList
+#
+# <b>A Hash subclass designed for mapping word lists to token types.</b>
+#
+# Copyright (c) 2006 by murphy (Kornelius Kalnbach) <murphy rubychan de>
+#
+# License:: LGPL / ask the author
+# Version:: 1.1 (2006-Oct-19)
+#
+# A WordList is a Hash with some additional features.
+# It is intended to be used for keyword recognition.
+#
+# WordList is highly optimized to be used in Scanners,
+# typically to decide whether a given ident is a special token.
+#
+# For case insensitive words use CaseIgnoringWordList.
+#
+# Example:
+#
+# # define word arrays
+# RESERVED_WORDS = %w[
+# asm break case continue default do else
+# ...
+# ]
+#
+# PREDEFINED_TYPES = %w[
+# int long short char void
+# ...
+# ]
+#
+# PREDEFINED_CONSTANTS = %w[
+# EOF NULL ...
+# ]
+#
+# # make a WordList
+# IDENT_KIND = WordList.new(:ident).
+# add(RESERVED_WORDS, :reserved).
+# add(PREDEFINED_TYPES, :pre_type).
+# add(PREDEFINED_CONSTANTS, :pre_constant)
+#
+# ...
+#
+# def scan_tokens tokens, options
+# ...
+#
+# elsif scan(/[A-Za-z_][A-Za-z_0-9]*/)
+# # use it
+# kind = IDENT_KIND[match]
+# ...
+class WordList < Hash
+
+ # Creates a new WordList with +default+ as default value.
+ #
+ # You can activate +caching+ to store the results for every [] request.
+ #
+ # With caching, methods like +include?+ or +delete+ may no longer behave
+ # as you expect. Therefore, it is recommended to use the [] method only.
+ def initialize default = false, caching = false, &block
+ if block
+ raise ArgumentError, 'Can\'t combine block with caching.' if caching
+ super(&block)
+ else
+ if caching
+ super() do |h, k|
+ h[k] = h.fetch k, default
+ end
+ else
+ super default
+ end
+ end
+ end
+
+ # Add words to the list and associate them with +kind+.
+ #
+ # Returns +self+, so you can concat add calls.
+ def add words, kind = true
+ words.each do |word|
+ self[word] = kind
+ end
+ self
+ end
+
+end
+
+
+# A CaseIgnoringWordList is like a WordList, only that
+# keys are compared case-insensitively.
+#
+# Ignoring the text case is realized by sending the +downcase+ message to
+# all keys.
+#
+# Caching usually makes a CaseIgnoringWordList faster, but it has to be
+# activated explicitely.
+class CaseIgnoringWordList < WordList
+
+ # Creates a new case-insensitive WordList with +default+ as default value.
+ #
+ # You can activate caching to store the results for every [] request.
+ # This speeds up subsequent lookups for the same word, but also
+ # uses memory.
+ def initialize default = false, caching = false
+ if caching
+ super(default, false) do |h, k|
+ h[k] = h.fetch k.downcase, default
+ end
+ else
+ super(default, false)
+ extend Uncached
+ end
+ end
+
+ module Uncached # :nodoc:
+ def [] key
+ super(key.downcase)
+ end
+ end
+
+ # Add +words+ to the list and associate them with +kind+.
+ def add words, kind = true
+ words.each do |word|
+ self[word.downcase] = kind
+ end
+ self
+ end
+
+end
+
+end
+
+__END__
+# check memory consumption
+END {
+ ObjectSpace.each_object(CodeRay::CaseIgnoringWordList) do |wl|
+ p wl.inject(0) { |memo, key, value| memo + key.size + 24 }
+ end
+}
\ No newline at end of file
--- /dev/null
+module CodeRay
+
+ require 'coderay/helpers/plugin'
+
+ # = Scanners
+ #
+ # This module holds the Scanner class and its subclasses.
+ # For example, the Ruby scanner is named CodeRay::Scanners::Ruby
+ # can be found in coderay/scanners/ruby.
+ #
+ # Scanner also provides methods and constants for the register
+ # mechanism and the [] method that returns the Scanner class
+ # belonging to the given lang.
+ #
+ # See PluginHost.
+ module Scanners
+ extend PluginHost
+ plugin_path File.dirname(__FILE__), 'scanners'
+
+ require 'strscan'
+
+ # = Scanner
+ #
+ # The base class for all Scanners.
+ #
+ # It is a subclass of Ruby's great +StringScanner+, which
+ # makes it easy to access the scanning methods inside.
+ #
+ # It is also +Enumerable+, so you can use it like an Array of
+ # Tokens:
+ #
+ # require 'coderay'
+ #
+ # c_scanner = CodeRay::Scanners[:c].new "if (*p == '{') nest++;"
+ #
+ # for text, kind in c_scanner
+ # puts text if kind == :operator
+ # end
+ #
+ # # prints: (*==)++;
+ #
+ # OK, this is a very simple example :)
+ # You can also use +map+, +any?+, +find+ and even +sort_by+,
+ # if you want.
+ class Scanner < StringScanner
+
+ extend Plugin
+ plugin_host Scanners
+
+ # Raised if a Scanner fails while scanning
+ ScanError = Class.new(Exception)
+
+ require 'coderay/helpers/word_list'
+
+ # The default options for all scanner classes.
+ #
+ # Define @default_options for subclasses.
+ DEFAULT_OPTIONS = { :stream => false }
+
+ KINDS_NOT_LOC = [:comment, :doctype]
+
+ class << self
+
+ # Returns if the Scanner can be used in streaming mode.
+ def streamable?
+ is_a? Streamable
+ end
+
+ def normify code
+ code = code.to_s
+ if code.respond_to?(:encoding) && (code.encoding.name != 'UTF-8' || !code.valid_encoding?)
+ code = code.dup
+ original_encoding = code.encoding
+ code.force_encoding 'Windows-1252'
+ unless code.valid_encoding?
+ code.force_encoding original_encoding
+ if code.encoding.name == 'UTF-8'
+ code.encode! 'UTF-16BE', :invalid => :replace, :undef => :replace, :replace => '?'
+ end
+ code.encode! 'UTF-8', :invalid => :replace, :undef => :replace, :replace => '?'
+ end
+ end
+ code.to_unix
+ end
+
+ def file_extension extension = nil
+ if extension
+ @file_extension = extension.to_s
+ else
+ @file_extension ||= plugin_id.to_s
+ end
+ end
+
+ end
+
+=begin
+## Excluded for speed reasons; protected seems to make methods slow.
+
+ # Save the StringScanner methods from being called.
+ # This would not be useful for highlighting.
+ strscan_public_methods =
+ StringScanner.instance_methods -
+ StringScanner.ancestors[1].instance_methods
+ protected(*strscan_public_methods)
+=end
+
+ # Create a new Scanner.
+ #
+ # * +code+ is the input String and is handled by the superclass
+ # StringScanner.
+ # * +options+ is a Hash with Symbols as keys.
+ # It is merged with the default options of the class (you can
+ # overwrite default options here.)
+ # * +block+ is the callback for streamed highlighting.
+ #
+ # If you set :stream to +true+ in the options, the Scanner uses a
+ # TokenStream with the +block+ as callback to handle the tokens.
+ #
+ # Else, a Tokens object is used.
+ def initialize code='', options = {}, &block
+ raise "I am only the basic Scanner class. I can't scan "\
+ "anything. :( Use my subclasses." if self.class == Scanner
+
+ @options = self.class::DEFAULT_OPTIONS.merge options
+
+ super Scanner.normify(code)
+
+ @tokens = options[:tokens]
+ if @options[:stream]
+ warn "warning in CodeRay::Scanner.new: :stream is set, "\
+ "but no block was given" unless block_given?
+ raise NotStreamableError, self unless kind_of? Streamable
+ @tokens ||= TokenStream.new(&block)
+ else
+ warn "warning in CodeRay::Scanner.new: Block given, "\
+ "but :stream is #{@options[:stream]}" if block_given?
+ @tokens ||= Tokens.new
+ end
+ @tokens.scanner = self
+
+ setup
+ end
+
+ def reset
+ super
+ reset_instance
+ end
+
+ def string= code
+ code = Scanner.normify(code)
+ if defined?(RUBY_DESCRIPTION) && RUBY_DESCRIPTION['rubinius 1.0.1']
+ reset_state
+ @string = code
+ else
+ super code
+ end
+ reset_instance
+ end
+
+ # More mnemonic accessor name for the input string.
+ alias code string
+ alias code= string=
+
+ # Returns the Plugin ID for this scanner.
+ def lang
+ self.class.plugin_id
+ end
+
+ # Scans the code and returns all tokens in a Tokens object.
+ def tokenize new_string=nil, options = {}
+ options = @options.merge(options)
+ self.string = new_string if new_string
+ @cached_tokens =
+ if @options[:stream] # :stream must have been set already
+ reset unless new_string
+ scan_tokens @tokens, options
+ @tokens
+ else
+ scan_tokens @tokens, options
+ end
+ end
+
+ def tokens
+ @cached_tokens ||= tokenize
+ end
+
+ # Whether the scanner is in streaming mode.
+ def streaming?
+ !!@options[:stream]
+ end
+
+ # Traverses the tokens.
+ def each &block
+ raise ArgumentError,
+ 'Cannot traverse TokenStream.' if @options[:stream]
+ tokens.each(&block)
+ end
+ include Enumerable
+
+ # The current line position of the scanner.
+ #
+ # Beware, this is implemented inefficiently. It should be used
+ # for debugging only.
+ def line
+ string[0..pos].count("\n") + 1
+ end
+
+ def column pos = self.pos
+ return 0 if pos <= 0
+ string = string()
+ if string.respond_to?(:bytesize) && (defined?(@bin_string) || string.bytesize != string.size)
+ @bin_string ||= string.dup.force_encoding('binary')
+ string = @bin_string
+ end
+ pos - (string.rindex(?\n, pos) || 0)
+ end
+
+ def marshal_dump
+ @options
+ end
+
+ def marshal_load options
+ @options = options
+ end
+
+ protected
+
+ # Can be implemented by subclasses to do some initialization
+ # that has to be done once per instance.
+ #
+ # Use reset for initialization that has to be done once per
+ # scan.
+ def setup
+ end
+
+ # This is the central method, and commonly the only one a
+ # subclass implements.
+ #
+ # Subclasses must implement this method; it must return +tokens+
+ # and must only use Tokens#<< for storing scanned tokens!
+ def scan_tokens tokens, options
+ raise NotImplementedError,
+ "#{self.class}#scan_tokens not implemented."
+ end
+
+ def reset_instance
+ @tokens.clear unless @options[:keep_tokens]
+ @cached_tokens = nil
+ @bin_string = nil if defined? @bin_string
+ end
+
+ # Scanner error with additional status information
+ def raise_inspect msg, tokens, state = 'No state given!', ambit = 30
+ raise ScanError, <<-EOE % [
+
+
+***ERROR in %s: %s (after %d tokens)
+
+tokens:
+%s
+
+current line: %d column: %d pos: %d
+matched: %p state: %p
+bol? = %p, eos? = %p
+
+surrounding code:
+%p ~~ %p
+
+
+***ERROR***
+
+ EOE
+ File.basename(caller[0]),
+ msg,
+ tokens.size,
+ tokens.last(10).map { |t| t.inspect }.join("\n"),
+ line, column, pos,
+ matched, state, bol?, eos?,
+ string[pos - ambit, ambit],
+ string[pos, ambit],
+ ]
+ end
+
+ end
+
+ end
+end
+
+class String
+ # I love this hack. It seems to silence all dos/unix/mac newline problems.
+ def to_unix
+ if index ?\r
+ gsub(/\r\n?/, "\n")
+ else
+ self
+ end
+ end
+end
--- /dev/null
+module CodeRay
+module Scanners
+
+ map \
+ :h => :c,
+ :cplusplus => :cpp,
+ :'c++' => :cpp,
+ :ecma => :java_script,
+ :ecmascript => :java_script,
+ :ecma_script => :java_script,
+ :irb => :ruby,
+ :javascript => :java_script,
+ :js => :java_script,
+ :nitro => :nitro_xhtml,
+ :pascal => :delphi,
+ :plain => :plaintext,
+ :xhtml => :html,
+ :yml => :yaml
+
+ default :plain
+
+end
+end
--- /dev/null
+module CodeRay
+module Scanners
+
+ class C < Scanner
+
+ include Streamable
+
+ register_for :c
+ file_extension 'c'
+
+ RESERVED_WORDS = [
+ 'asm', 'break', 'case', 'continue', 'default', 'do',
+ 'else', 'enum', 'for', 'goto', 'if', 'return',
+ 'sizeof', 'struct', 'switch', 'typedef', 'union', 'while',
+ 'restrict', # added in C99
+ ]
+
+ PREDEFINED_TYPES = [
+ 'int', 'long', 'short', 'char',
+ 'signed', 'unsigned', 'float', 'double',
+ 'bool', 'complex', # added in C99
+ ]
+
+ PREDEFINED_CONSTANTS = [
+ 'EOF', 'NULL',
+ 'true', 'false', # added in C99
+ ]
+ DIRECTIVES = [
+ 'auto', 'extern', 'register', 'static', 'void',
+ 'const', 'volatile', # added in C89
+ 'inline', # added in C99
+ ]
+
+ IDENT_KIND = WordList.new(:ident).
+ add(RESERVED_WORDS, :reserved).
+ add(PREDEFINED_TYPES, :pre_type).
+ add(DIRECTIVES, :directive).
+ add(PREDEFINED_CONSTANTS, :pre_constant)
+
+ ESCAPE = / [rbfntv\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} /x
+ UNICODE_ESCAPE = / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} /x
+
+ def scan_tokens tokens, options
+
+ state = :initial
+ label_expected = true
+ case_expected = false
+ label_expected_before_preproc_line = nil
+ in_preproc_line = false
+
+ until eos?
+
+ kind = nil
+ match = nil
+
+ case state
+
+ when :initial
+
+ if match = scan(/ \s+ | \\\n /x)
+ if in_preproc_line && match != "\\\n" && match.index(?\n)
+ in_preproc_line = false
+ label_expected = label_expected_before_preproc_line
+ end
+ tokens << [match, :space]
+ next
+
+ elsif scan(%r! // [^\n\\]* (?: \\. [^\n\\]* )* | /\* (?: .*? \*/ | .* ) !mx)
+ kind = :comment
+
+ elsif match = scan(/ \# \s* if \s* 0 /x)
+ match << scan_until(/ ^\# (?:elif|else|endif) .*? $ | \z /xm) unless eos?
+ kind = :comment
+
+ elsif match = scan(/ [-+*=<>?:;,!&^|()\[\]{}~%]+ | \/=? | \.(?!\d) /x)
+ label_expected = match =~ /[;\{\}]/
+ if case_expected
+ label_expected = true if match == ':'
+ case_expected = false
+ end
+ kind = :operator
+
+ elsif match = scan(/ [A-Za-z_][A-Za-z_0-9]* /x)
+ kind = IDENT_KIND[match]
+ if kind == :ident && label_expected && !in_preproc_line && scan(/:(?!:)/)
+ kind = :label
+ match << matched
+ else
+ label_expected = false
+ if kind == :reserved
+ case match
+ when 'case', 'default'
+ case_expected = true
+ end
+ end
+ end
+
+ elsif scan(/\$/)
+ kind = :ident
+
+ elsif match = scan(/L?"/)
+ tokens << [:open, :string]
+ if match[0] == ?L
+ tokens << ['L', :modifier]
+ match = '"'
+ end
+ state = :string
+ kind = :delimiter
+
+ elsif scan(/#[ \t]*(\w*)/)
+ kind = :preprocessor
+ in_preproc_line = true
+ label_expected_before_preproc_line = label_expected
+ state = :include_expected if self[1] == 'include'
+
+ elsif scan(/ L?' (?: [^\'\n\\] | \\ #{ESCAPE} )? '? /ox)
+ label_expected = false
+ kind = :char
+
+ elsif scan(/0[xX][0-9A-Fa-f]+/)
+ label_expected = false
+ kind = :hex
+
+ elsif scan(/(?:0[0-7]+)(?![89.eEfF])/)
+ label_expected = false
+ kind = :oct
+
+ elsif scan(/(?:\d+)(?![.eEfF])L?L?/)
+ label_expected = false
+ kind = :integer
+
+ elsif scan(/\d[fF]?|\d*\.\d+(?:[eE][+-]?\d+)?[fF]?|\d+[eE][+-]?\d+[fF]?/)
+ label_expected = false
+ kind = :float
+
+ else
+ getch
+ kind = :error
+
+ end
+
+ when :string
+ if scan(/[^\\\n"]+/)
+ kind = :content
+ elsif scan(/"/)
+ tokens << ['"', :delimiter]
+ tokens << [:close, :string]
+ state = :initial
+ label_expected = false
+ next
+ elsif scan(/ \\ (?: #{ESCAPE} | #{UNICODE_ESCAPE} ) /mox)
+ kind = :char
+ elsif scan(/ \\ | $ /x)
+ tokens << [:close, :string]
+ kind = :error
+ state = :initial
+ label_expected = false
+ else
+ raise_inspect "else case \" reached; %p not handled." % peek(1), tokens
+ end
+
+ when :include_expected
+ if scan(/<[^>\n]+>?|"[^"\n\\]*(?:\\.[^"\n\\]*)*"?/)
+ kind = :include
+ state = :initial
+
+ elsif match = scan(/\s+/)
+ kind = :space
+ state = :initial if match.index ?\n
+
+ else
+ state = :initial
+ next
+
+ end
+
+ else
+ raise_inspect 'Unknown state', tokens
+
+ end
+
+ match ||= matched
+ if $CODERAY_DEBUG and not kind
+ raise_inspect 'Error token %p in line %d' %
+ [[match, kind], line], tokens
+ end
+ raise_inspect 'Empty token', tokens unless match
+
+ tokens << [match, kind]
+
+ end
+
+ if state == :string
+ tokens << [:close, :string]
+ end
+
+ tokens
+ end
+
+ end
+
+end
+end
--- /dev/null
+module CodeRay
+module Scanners
+
+ class CPlusPlus < Scanner
+
+ include Streamable
+
+ register_for :cpp
+ file_extension 'cpp'
+ title 'C++'
+
+ # http://www.cppreference.com/wiki/keywords/start
+ RESERVED_WORDS = [
+ 'and', 'and_eq', 'asm', 'bitand', 'bitor', 'break',
+ 'case', 'catch', 'class', 'compl', 'const_cast',
+ 'continue', 'default', 'delete', 'do', 'dynamic_cast', 'else',
+ 'enum', 'export', 'for', 'goto', 'if', 'namespace', 'new',
+ 'not', 'not_eq', 'or', 'or_eq', 'reinterpret_cast', 'return',
+ 'sizeof', 'static_cast', 'struct', 'switch', 'template',
+ 'throw', 'try', 'typedef', 'typeid', 'typename', 'union',
+ 'while', 'xor', 'xor_eq'
+ ]
+
+ PREDEFINED_TYPES = [
+ 'bool', 'char', 'double', 'float', 'int', 'long',
+ 'short', 'signed', 'unsigned', 'wchar_t', 'string'
+ ]
+ PREDEFINED_CONSTANTS = [
+ 'false', 'true',
+ 'EOF', 'NULL',
+ ]
+ PREDEFINED_VARIABLES = [
+ 'this'
+ ]
+ DIRECTIVES = [
+ 'auto', 'const', 'explicit', 'extern', 'friend', 'inline', 'mutable', 'operator',
+ 'private', 'protected', 'public', 'register', 'static', 'using', 'virtual', 'void',
+ 'volatile'
+ ]
+
+ IDENT_KIND = WordList.new(:ident).
+ add(RESERVED_WORDS, :reserved).
+ add(PREDEFINED_TYPES, :pre_type).
+ add(PREDEFINED_VARIABLES, :local_variable).
+ add(DIRECTIVES, :directive).
+ add(PREDEFINED_CONSTANTS, :pre_constant)
+
+ ESCAPE = / [rbfntv\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} /x
+ UNICODE_ESCAPE = / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} /x
+
+ def scan_tokens tokens, options
+
+ state = :initial
+ label_expected = true
+ case_expected = false
+ label_expected_before_preproc_line = nil
+ in_preproc_line = false
+
+ until eos?
+
+ kind = nil
+ match = nil
+
+ case state
+
+ when :initial
+
+ if match = scan(/ \s+ | \\\n /x)
+ if in_preproc_line && match != "\\\n" && match.index(?\n)
+ in_preproc_line = false
+ label_expected = label_expected_before_preproc_line
+ end
+ tokens << [match, :space]
+ next
+
+ elsif scan(%r! // [^\n\\]* (?: \\. [^\n\\]* )* | /\* (?: .*? \*/ | .* ) !mx)
+ kind = :comment
+
+ elsif match = scan(/ \# \s* if \s* 0 /x)
+ match << scan_until(/ ^\# (?:elif|else|endif) .*? $ | \z /xm) unless eos?
+ kind = :comment
+
+ elsif match = scan(/ [-+*=<>?:;,!&^|()\[\]{}~%]+ | \/=? | \.(?!\d) /x)
+ label_expected = match =~ /[;\{\}]/
+ if case_expected
+ label_expected = true if match == ':'
+ case_expected = false
+ end
+ kind = :operator
+
+ elsif match = scan(/ [A-Za-z_][A-Za-z_0-9]* /x)
+ kind = IDENT_KIND[match]
+ if kind == :ident && label_expected && !in_preproc_line && scan(/:(?!:)/)
+ kind = :label
+ match << matched
+ else
+ label_expected = false
+ if kind == :reserved
+ case match
+ when 'class'
+ state = :class_name_expected
+ when 'case', 'default'
+ case_expected = true
+ end
+ end
+ end
+
+ elsif scan(/\$/)
+ kind = :ident
+
+ elsif match = scan(/L?"/)
+ tokens << [:open, :string]
+ if match[0] == ?L
+ tokens << ['L', :modifier]
+ match = '"'
+ end
+ state = :string
+ kind = :delimiter
+
+ elsif scan(/#[ \t]*(\w*)/)
+ kind = :preprocessor
+ in_preproc_line = true
+ label_expected_before_preproc_line = label_expected
+ state = :include_expected if self[1] == 'include'
+
+ elsif scan(/ L?' (?: [^\'\n\\] | \\ #{ESCAPE} )? '? /ox)
+ label_expected = false
+ kind = :char
+
+ elsif scan(/0[xX][0-9A-Fa-f]+/)
+ label_expected = false
+ kind = :hex
+
+ elsif scan(/(?:0[0-7]+)(?![89.eEfF])/)
+ label_expected = false
+ kind = :oct
+
+ elsif scan(/(?:\d+)(?![.eEfF])L?L?/)
+ label_expected = false
+ kind = :integer
+
+ elsif scan(/\d[fF]?|\d*\.\d+(?:[eE][+-]?\d+)?[fF]?|\d+[eE][+-]?\d+[fF]?/)
+ label_expected = false
+ kind = :float
+
+ else
+ getch
+ kind = :error
+
+ end
+
+ when :string
+ if scan(/[^\\"]+/)
+ kind = :content
+ elsif scan(/"/)
+ tokens << ['"', :delimiter]
+ tokens << [:close, :string]
+ state = :initial
+ label_expected = false
+ next
+ elsif scan(/ \\ (?: #{ESCAPE} | #{UNICODE_ESCAPE} ) /mox)
+ kind = :char
+ elsif scan(/ \\ | $ /x)
+ tokens << [:close, :string]
+ kind = :error
+ state = :initial
+ label_expected = false
+ else
+ raise_inspect "else case \" reached; %p not handled." % peek(1), tokens
+ end
+
+ when :include_expected
+ if scan(/<[^>\n]+>?|"[^"\n\\]*(?:\\.[^"\n\\]*)*"?/)
+ kind = :include
+ state = :initial
+
+ elsif match = scan(/\s+/)
+ kind = :space
+ state = :initial if match.index ?\n
+
+ else
+ state = :initial
+ next
+
+ end
+
+ when :class_name_expected
+ if scan(/ [A-Za-z_][A-Za-z_0-9]* /x)
+ kind = :class
+ state = :initial
+
+ elsif match = scan(/\s+/)
+ kind = :space
+
+ else
+ getch
+ kind = :error
+ state = :initial
+
+ end
+
+ else
+ raise_inspect 'Unknown state', tokens
+
+ end
+
+ match ||= matched
+ if $CODERAY_DEBUG and not kind
+ raise_inspect 'Error token %p in line %d' %
+ [[match, kind], line], tokens
+ end
+ raise_inspect 'Empty token', tokens unless match
+
+ tokens << [match, kind]
+
+ end
+
+ if state == :string
+ tokens << [:close, :string]
+ end
+
+ tokens
+ end
+
+ end
+
+end
+end
--- /dev/null
+module CodeRay
+module Scanners
+
+ class CSS < Scanner
+
+ register_for :css
+
+ KINDS_NOT_LOC = [
+ :comment,
+ :class, :pseudo_class, :type,
+ :constant, :directive,
+ :key, :value, :operator, :color, :float,
+ :error, :important,
+ ]
+
+ module RE
+ Hex = /[0-9a-fA-F]/
+ Unicode = /\\#{Hex}{1,6}(?:\r\n|\s)?/ # differs from standard because it allows uppercase hex too
+ Escape = /#{Unicode}|\\[^\r\n\f0-9a-fA-F]/
+ NMChar = /[-_a-zA-Z0-9]|#{Escape}/
+ NMStart = /[_a-zA-Z]|#{Escape}/
+ NL = /\r\n|\r|\n|\f/
+ String1 = /"(?:[^\n\r\f\\"]|\\#{NL}|#{Escape})*"?/ # FIXME: buggy regexp
+ String2 = /'(?:[^\n\r\f\\']|\\#{NL}|#{Escape})*'?/ # FIXME: buggy regexp
+ String = /#{String1}|#{String2}/
+
+ HexColor = /#(?:#{Hex}{6}|#{Hex}{3})/
+ Color = /#{HexColor}/
+
+ Num = /-?(?:[0-9]+|[0-9]*\.[0-9]+)/
+ Name = /#{NMChar}+/
+ Ident = /-?#{NMStart}#{NMChar}*/
+ AtKeyword = /@#{Ident}/
+ Percentage = /#{Num}%/
+
+ reldimensions = %w[em ex px]
+ absdimensions = %w[in cm mm pt pc]
+ Unit = Regexp.union(*(reldimensions + absdimensions))
+
+ Dimension = /#{Num}#{Unit}/
+
+ Comment = %r! /\* (?: .*? \*/ | .* ) !mx
+ Function = /(?:url|alpha)\((?:[^)\n\r\f]|\\\))*\)?/
+
+ Id = /##{Name}/
+ Class = /\.#{Name}/
+ PseudoClass = /:#{Name}/
+ AttributeSelector = /\[[^\]]*\]?/
+
+ end
+
+ def scan_tokens tokens, options
+
+ value_expected = nil
+ states = [:initial]
+
+ until eos?
+
+ kind = nil
+ match = nil
+
+ if scan(/\s+/)
+ kind = :space
+
+ elsif case states.last
+ when :initial, :media
+ if scan(/(?>#{RE::Ident})(?!\()|\*/ox)
+ kind = :type
+ elsif scan RE::Class
+ kind = :class
+ elsif scan RE::Id
+ kind = :constant
+ elsif scan RE::PseudoClass
+ kind = :pseudo_class
+ elsif match = scan(RE::AttributeSelector)
+ # TODO: Improve highlighting inside of attribute selectors.
+ tokens << [:open, :string]
+ tokens << [match[0,1], :delimiter]
+ tokens << [match[1..-2], :content] if match.size > 2
+ tokens << [match[-1,1], :delimiter] if match[-1] == ?]
+ tokens << [:close, :string]
+ next
+ elsif match = scan(/@media/)
+ kind = :directive
+ states.push :media_before_name
+ end
+
+ when :block
+ if scan(/(?>#{RE::Ident})(?!\()/ox)
+ if value_expected
+ kind = :value
+ else
+ kind = :key
+ end
+ end
+
+ when :media_before_name
+ if scan RE::Ident
+ kind = :type
+ states[-1] = :media_after_name
+ end
+
+ when :media_after_name
+ if scan(/\{/)
+ kind = :operator
+ states[-1] = :media
+ end
+
+ when :comment
+ if scan(/(?:[^*\s]|\*(?!\/))+/)
+ kind = :comment
+ elsif scan(/\*\//)
+ kind = :comment
+ states.pop
+ elsif scan(/\s+/)
+ kind = :space
+ end
+
+ else
+ raise_inspect 'Unknown state', tokens
+
+ end
+
+ elsif scan(/\/\*/)
+ kind = :comment
+ states.push :comment
+
+ elsif scan(/\{/)
+ value_expected = false
+ kind = :operator
+ states.push :block
+
+ elsif scan(/\}/)
+ value_expected = false
+ if states.last == :block || states.last == :media
+ kind = :operator
+ states.pop
+ else
+ kind = :error
+ end
+
+ elsif match = scan(/#{RE::String}/o)
+ tokens << [:open, :string]
+ tokens << [match[0, 1], :delimiter]
+ tokens << [match[1..-2], :content] if match.size > 2
+ tokens << [match[-1, 1], :delimiter] if match.size >= 2
+ tokens << [:close, :string]
+ next
+
+ elsif match = scan(/#{RE::Function}/o)
+ tokens << [:open, :string]
+ start = match[/^\w+\(/]
+ tokens << [start, :delimiter]
+ if match[-1] == ?)
+ tokens << [match[start.size..-2], :content]
+ tokens << [')', :delimiter]
+ else
+ tokens << [match[start.size..-1], :content]
+ end
+ tokens << [:close, :string]
+ next
+
+ elsif scan(/(?: #{RE::Dimension} | #{RE::Percentage} | #{RE::Num} )/ox)
+ kind = :float
+
+ elsif scan(/#{RE::Color}/o)
+ kind = :color
+
+ elsif scan(/! *important/)
+ kind = :important
+
+ elsif scan(/rgb\([^()\n]*\)?/)
+ kind = :color
+
+ elsif scan(/#{RE::AtKeyword}/o)
+ kind = :directive
+
+ elsif match = scan(/ [+>:;,.=()\/] /x)
+ if match == ':'
+ value_expected = true
+ elsif match == ';'
+ value_expected = false
+ end
+ kind = :operator
+
+ else
+ getch
+ kind = :error
+
+ end
+
+ match ||= matched
+ if $CODERAY_DEBUG and not kind
+ raise_inspect 'Error token %p in line %d' %
+ [[match, kind], line], tokens
+ end
+ raise_inspect 'Empty token', tokens unless match
+
+ tokens << [match, kind]
+
+ end
+
+ tokens
+ end
+
+ end
+
+end
+end
--- /dev/null
+module CodeRay
+module Scanners
+
+ # = Debug Scanner
+ class Debug < Scanner
+
+ include Streamable
+ register_for :debug
+ file_extension 'raydebug'
+ title 'CodeRay Token Dump'
+
+ protected
+ def scan_tokens tokens, options
+
+ opened_tokens = []
+
+ until eos?
+
+ kind = nil
+ match = nil
+
+ if scan(/\s+/)
+ tokens << [matched, :space]
+ next
+
+ elsif scan(/ (\w+) \( ( [^\)\\]* ( \\. [^\)\\]* )* ) \) /x)
+ kind = self[1].to_sym
+ match = self[2].gsub(/\\(.)/, '\1')
+
+ elsif scan(/ (\w+) < /x)
+ kind = self[1].to_sym
+ opened_tokens << kind
+ match = :open
+
+ elsif !opened_tokens.empty? && scan(/ > /x)
+ kind = opened_tokens.pop || :error
+ match = :close
+
+ else
+ kind = :error
+ getch
+
+ end
+
+ match ||= matched
+ if $CODERAY_DEBUG and not kind
+ raise_inspect 'Error token %p in line %d' %
+ [[match, kind], line], tokens
+ end
+ raise_inspect 'Empty token', tokens unless match
+
+ tokens << [match, kind]
+
+ end
+
+ tokens
+ end
+
+ end
+
+end
+end
--- /dev/null
+module CodeRay
+module Scanners
+
+ class Delphi < Scanner
+
+ register_for :delphi
+ file_extension 'pas'
+
+ RESERVED_WORDS = [
+ 'and', 'array', 'as', 'at', 'asm', 'at', 'begin', 'case', 'class',
+ 'const', 'constructor', 'destructor', 'dispinterface', 'div', 'do',
+ 'downto', 'else', 'end', 'except', 'exports', 'file', 'finalization',
+ 'finally', 'for', 'function', 'goto', 'if', 'implementation', 'in',
+ 'inherited', 'initialization', 'inline', 'interface', 'is', 'label',
+ 'library', 'mod', 'nil', 'not', 'object', 'of', 'or', 'out', 'packed',
+ 'procedure', 'program', 'property', 'raise', 'record', 'repeat',
+ 'resourcestring', 'set', 'shl', 'shr', 'string', 'then', 'threadvar',
+ 'to', 'try', 'type', 'unit', 'until', 'uses', 'var', 'while', 'with',
+ 'xor', 'on'
+ ]
+
+ DIRECTIVES = [
+ 'absolute', 'abstract', 'assembler', 'at', 'automated', 'cdecl',
+ 'contains', 'deprecated', 'dispid', 'dynamic', 'export',
+ 'external', 'far', 'forward', 'implements', 'local',
+ 'near', 'nodefault', 'on', 'overload', 'override',
+ 'package', 'pascal', 'platform', 'private', 'protected', 'public',
+ 'published', 'read', 'readonly', 'register', 'reintroduce',
+ 'requires', 'resident', 'safecall', 'stdcall', 'stored', 'varargs',
+ 'virtual', 'write', 'writeonly'
+ ]
+
+ IDENT_KIND = CaseIgnoringWordList.new(:ident).
+ add(RESERVED_WORDS, :reserved).
+ add(DIRECTIVES, :directive)
+
+ NAME_FOLLOWS = CaseIgnoringWordList.new(false).
+ add(%w(procedure function .))
+
+ private
+ def scan_tokens tokens, options
+
+ state = :initial
+ last_token = ''
+
+ until eos?
+
+ kind = nil
+ match = nil
+
+ if state == :initial
+
+ if scan(/ \s+ /x)
+ tokens << [matched, :space]
+ next
+
+ elsif scan(%r! \{ \$ [^}]* \}? | \(\* \$ (?: .*? \*\) | .* ) !mx)
+ tokens << [matched, :preprocessor]
+ next
+
+ elsif scan(%r! // [^\n]* | \{ [^}]* \}? | \(\* (?: .*? \*\) | .* ) !mx)
+ tokens << [matched, :comment]
+ next
+
+ elsif match = scan(/ <[>=]? | >=? | :=? | [-+=*\/;,@\^|\(\)\[\]] | \.\. /x)
+ kind = :operator
+
+ elsif match = scan(/\./)
+ kind = :operator
+ if last_token == 'end'
+ tokens << [match, kind]
+ next
+ end
+
+ elsif match = scan(/ [A-Za-z_][A-Za-z_0-9]* /x)
+ kind = NAME_FOLLOWS[last_token] ? :ident : IDENT_KIND[match]
+
+ elsif match = scan(/ ' ( [^\n']|'' ) (?:'|$) /x)
+ tokens << [:open, :char]
+ tokens << ["'", :delimiter]
+ tokens << [self[1], :content]
+ tokens << ["'", :delimiter]
+ tokens << [:close, :char]
+ next
+
+ elsif match = scan(/ ' /x)
+ tokens << [:open, :string]
+ state = :string
+ kind = :delimiter
+
+ elsif scan(/ \# (?: \d+ | \$[0-9A-Fa-f]+ ) /x)
+ kind = :char
+
+ elsif scan(/ \$ [0-9A-Fa-f]+ /x)
+ kind = :hex
+
+ elsif scan(/ (?: \d+ ) (?![eE]|\.[^.]) /x)
+ kind = :integer
+
+ elsif scan(/ \d+ (?: \.\d+ (?: [eE][+-]? \d+ )? | [eE][+-]? \d+ ) /x)
+ kind = :float
+
+ else
+ kind = :error
+ getch
+
+ end
+
+ elsif state == :string
+ if scan(/[^\n']+/)
+ kind = :content
+ elsif scan(/''/)
+ kind = :char
+ elsif scan(/'/)
+ tokens << ["'", :delimiter]
+ tokens << [:close, :string]
+ state = :initial
+ next
+ elsif scan(/\n/)
+ tokens << [:close, :string]
+ kind = :error
+ state = :initial
+ else
+ raise "else case \' reached; %p not handled." % peek(1), tokens
+ end
+
+ else
+ raise 'else-case reached', tokens
+
+ end
+
+ match ||= matched
+ if $CODERAY_DEBUG and not kind
+ raise_inspect 'Error token %p in line %d' %
+ [[match, kind], line], tokens, state
+ end
+ raise_inspect 'Empty token', tokens unless match
+
+ last_token = match
+ tokens << [match, kind]
+
+ end
+
+ tokens
+ end
+
+ end
+
+end
+end
--- /dev/null
+module CodeRay
+module Scanners
+
+ class Diff < Scanner
+
+ register_for :diff
+ title 'diff output'
+
+ def scan_tokens tokens, options
+
+ line_kind = nil
+ state = :initial
+
+ until eos?
+ kind = match = nil
+
+ if match = scan(/\n/)
+ if line_kind
+ tokens << [:end_line, line_kind]
+ line_kind = nil
+ end
+ tokens << [match, :space]
+ next
+ end
+
+ case state
+
+ when :initial
+ if match = scan(/--- |\+\+\+ |=+|_+/)
+ tokens << [:begin_line, line_kind = :head]
+ tokens << [match, :head]
+ next unless match = scan(/.+/)
+ kind = :plain
+ elsif match = scan(/Index: |Property changes on: /)
+ tokens << [:begin_line, line_kind = :head]
+ tokens << [match, :head]
+ next unless match = scan(/.+/)
+ kind = :plain
+ elsif match = scan(/Added: /)
+ tokens << [:begin_line, line_kind = :head]
+ tokens << [match, :head]
+ next unless match = scan(/.+/)
+ kind = :plain
+ state = :added
+ elsif match = scan(/\\ /)
+ tokens << [:begin_line, line_kind = :change]
+ tokens << [match, :change]
+ next unless match = scan(/.+/)
+ kind = :plain
+ elsif match = scan(/@@(?>[^@\n]*)@@/)
+ if check(/\n|$/)
+ tokens << [:begin_line, line_kind = :change]
+ else
+ tokens << [:open, :change]
+ end
+ tokens << [match[0,2], :change]
+ tokens << [match[2...-2], :plain]
+ tokens << [match[-2,2], :change]
+ tokens << [:close, :change] unless line_kind
+ next unless match = scan(/.+/)
+ kind = :plain
+ elsif match = scan(/\+/)
+ tokens << [:begin_line, line_kind = :insert]
+ tokens << [match, :insert]
+ next unless match = scan(/.+/)
+ kind = :plain
+ elsif match = scan(/-/)
+ tokens << [:begin_line, line_kind = :delete]
+ tokens << [match, :delete]
+ next unless match = scan(/.+/)
+ kind = :plain
+ elsif scan(/ .*/)
+ kind = :comment
+ elsif scan(/.+/)
+ tokens << [:begin_line, line_kind = :comment]
+ kind = :plain
+ else
+ raise_inspect 'else case rached'
+ end
+
+ when :added
+ if match = scan(/ \+/)
+ tokens << [:begin_line, line_kind = :insert]
+ tokens << [match, :insert]
+ next unless match = scan(/.+/)
+ kind = :plain
+ else
+ state = :initial
+ next
+ end
+ end
+
+ match ||= matched
+ if $CODERAY_DEBUG and not kind
+ raise_inspect 'Error token %p in line %d' %
+ [[match, kind], line], tokens
+ end
+ raise_inspect 'Empty token', tokens unless match
+
+ tokens << [match, kind]
+ end
+
+ tokens << [:end_line, line_kind] if line_kind
+ tokens
+ end
+
+ end
+
+end
+end
--- /dev/null
+module CodeRay
+module Scanners
+
+ load :java
+
+ class Groovy < Java
+
+ include Streamable
+ register_for :groovy
+
+ # TODO: Check this!
+ GROOVY_KEYWORDS = %w[
+ as assert def in
+ ]
+ KEYWORDS_EXPECTING_VALUE = WordList.new.add %w[
+ case instanceof new return throw typeof while as assert in
+ ]
+ GROOVY_MAGIC_VARIABLES = %w[ it ]
+
+ IDENT_KIND = Java::IDENT_KIND.dup.
+ add(GROOVY_KEYWORDS, :keyword).
+ add(GROOVY_MAGIC_VARIABLES, :local_variable)
+
+ ESCAPE = / [bfnrtv$\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} /x
+ UNICODE_ESCAPE = / u[a-fA-F0-9]{4} /x # no 4-byte unicode chars? U[a-fA-F0-9]{8}
+ REGEXP_ESCAPE = / [bfnrtv\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} | \d | [bBdDsSwW\/] /x
+
+ # TODO: interpretation inside ', ", /
+ STRING_CONTENT_PATTERN = {
+ "'" => /(?>\\[^\\'\n]+|[^\\'\n]+)+/,
+ '"' => /[^\\$"\n]+/,
+ "'''" => /(?>[^\\']+|'(?!''))+/,
+ '"""' => /(?>[^\\$"]+|"(?!""))+/,
+ '/' => /[^\\$\/\n]+/,
+ }
+
+ def scan_tokens tokens, options
+
+ state = :initial
+ inline_block_stack = []
+ inline_block_paren_depth = nil
+ string_delimiter = nil
+ import_clause = class_name_follows = last_token = after_def = false
+ value_expected = true
+
+ until eos?
+
+ kind = nil
+ match = nil
+
+ case state
+
+ when :initial
+
+ if match = scan(/ \s+ | \\\n /x)
+ tokens << [match, :space]
+ if match.index ?\n
+ import_clause = after_def = false
+ value_expected = true unless value_expected
+ end
+ next
+
+ elsif scan(%r! // [^\n\\]* (?: \\. [^\n\\]* )* | /\* (?: .*? \*/ | .* ) !mx)
+ value_expected = true
+ after_def = false
+ kind = :comment
+
+ elsif bol? && scan(/ \#!.* /x)
+ kind = :doctype
+
+ elsif import_clause && scan(/ (?!as) #{IDENT} (?: \. #{IDENT} )* (?: \.\* )? /ox)
+ after_def = value_expected = false
+ kind = :include
+
+ elsif match = scan(/ #{IDENT} | \[\] /ox)
+ kind = IDENT_KIND[match]
+ value_expected = (kind == :keyword) && KEYWORDS_EXPECTING_VALUE[match]
+ if last_token == '.'
+ kind = :ident
+ elsif class_name_follows
+ kind = :class
+ class_name_follows = false
+ elsif after_def && check(/\s*[({]/)
+ kind = :method
+ after_def = false
+ elsif kind == :ident && last_token != '?' && check(/:/)
+ kind = :key
+ else
+ class_name_follows = true if match == 'class' || (import_clause && match == 'as')
+ import_clause = match == 'import'
+ after_def = true if match == 'def'
+ end
+
+ elsif scan(/;/)
+ import_clause = after_def = false
+ value_expected = true
+ kind = :operator
+
+ elsif scan(/\{/)
+ class_name_follows = after_def = false
+ value_expected = true
+ kind = :operator
+ if !inline_block_stack.empty?
+ inline_block_paren_depth += 1
+ end
+
+ # TODO: ~'...', ~"..." and ~/.../ style regexps
+ elsif match = scan(/ \.\.<? | \*?\.(?!\d)@? | \.& | \?:? | [,?:(\[] | -[->] | \+\+ |
+ && | \|\| | \*\*=? | ==?~ | <=?>? | [-+*%^~&|>=!]=? | <<<?=? | >>>?=? /x)
+ value_expected = true
+ value_expected = :regexp if match == '~'
+ after_def = false
+ kind = :operator
+
+ elsif match = scan(/ [)\]}] /x)
+ value_expected = after_def = false
+ if !inline_block_stack.empty? && match == '}'
+ inline_block_paren_depth -= 1
+ if inline_block_paren_depth == 0 # closing brace of inline block reached
+ tokens << [match, :inline_delimiter]
+ tokens << [:close, :inline]
+ state, string_delimiter, inline_block_paren_depth = inline_block_stack.pop
+ next
+ end
+ end
+ kind = :operator
+
+ elsif check(/[\d.]/)
+ after_def = value_expected = false
+ if scan(/0[xX][0-9A-Fa-f]+/)
+ kind = :hex
+ elsif scan(/(?>0[0-7]+)(?![89.eEfF])/)
+ kind = :oct
+ elsif scan(/\d+[fFdD]|\d*\.\d+(?:[eE][+-]?\d+)?[fFdD]?|\d+[eE][+-]?\d+[fFdD]?/)
+ kind = :float
+ elsif scan(/\d+[lLgG]?/)
+ kind = :integer
+ end
+
+ elsif match = scan(/'''|"""/)
+ after_def = value_expected = false
+ state = :multiline_string
+ tokens << [:open, :string]
+ string_delimiter = match
+ kind = :delimiter
+
+ # TODO: record.'name'
+ elsif match = scan(/["']/)
+ after_def = value_expected = false
+ state = match == '/' ? :regexp : :string
+ tokens << [:open, state]
+ string_delimiter = match
+ kind = :delimiter
+
+ elsif value_expected && (match = scan(/\//))
+ after_def = value_expected = false
+ tokens << [:open, :regexp]
+ state = :regexp
+ string_delimiter = '/'
+ kind = :delimiter
+
+ elsif scan(/ @ #{IDENT} /ox)
+ after_def = value_expected = false
+ kind = :annotation
+
+ elsif scan(/\//)
+ after_def = false
+ value_expected = true
+ kind = :operator
+
+ else
+ getch
+ kind = :error
+
+ end
+
+ when :string, :regexp, :multiline_string
+ if scan(STRING_CONTENT_PATTERN[string_delimiter])
+ kind = :content
+
+ elsif match = scan(state == :multiline_string ? /'''|"""/ : /["'\/]/)
+ tokens << [match, :delimiter]
+ if state == :regexp
+ # TODO: regexp modifiers? s, m, x, i?
+ modifiers = scan(/[ix]+/)
+ tokens << [modifiers, :modifier] if modifiers && !modifiers.empty?
+ end
+ state = :string if state == :multiline_string
+ tokens << [:close, state]
+ string_delimiter = nil
+ after_def = value_expected = false
+ state = :initial
+ next
+
+ elsif (state == :string || state == :multiline_string) &&
+ (match = scan(/ \\ (?: #{ESCAPE} | #{UNICODE_ESCAPE} ) /mox))
+ if string_delimiter[0] == ?' && !(match == "\\\\" || match == "\\'")
+ kind = :content
+ else
+ kind = :char
+ end
+ elsif state == :regexp && scan(/ \\ (?: #{REGEXP_ESCAPE} | #{UNICODE_ESCAPE} ) /mox)
+ kind = :char
+
+ elsif match = scan(/ \$ #{IDENT} /mox)
+ tokens << [:open, :inline]
+ tokens << ['$', :inline_delimiter]
+ match = match[1..-1]
+ tokens << [match, IDENT_KIND[match]]
+ tokens << [:close, :inline]
+ next
+ elsif match = scan(/ \$ \{ /x)
+ tokens << [:open, :inline]
+ tokens << ['${', :inline_delimiter]
+ inline_block_stack << [state, string_delimiter, inline_block_paren_depth]
+ inline_block_paren_depth = 1
+ state = :initial
+ next
+
+ elsif scan(/ \$ /mx)
+ kind = :content
+
+ elsif scan(/ \\. /mx)
+ kind = :content
+
+ elsif scan(/ \\ | \n /x)
+ tokens << [:close, state]
+ kind = :error
+ after_def = value_expected = false
+ state = :initial
+
+ else
+ raise_inspect "else case \" reached; %p not handled." % peek(1), tokens
+ end
+
+ else
+ raise_inspect 'Unknown state', tokens
+
+ end
+
+ match ||= matched
+ if $CODERAY_DEBUG and not kind
+ raise_inspect 'Error token %p in line %d' %
+ [[match, kind], line], tokens
+ end
+ raise_inspect 'Empty token', tokens unless match
+
+ last_token = match unless [:space, :comment, :doctype].include? kind
+
+ tokens << [match, kind]
+
+ end
+
+ if [:multiline_string, :string, :regexp].include? state
+ tokens << [:close, state]
+ end
+
+ tokens
+ end
+
+ end
+
+end
+end
--- /dev/null
+module CodeRay
+module Scanners
+
+ # HTML Scanner
+ class HTML < Scanner
+
+ include Streamable
+ register_for :html
+
+ KINDS_NOT_LOC = [
+ :comment, :doctype, :preprocessor,
+ :tag, :attribute_name, :operator,
+ :attribute_value, :delimiter, :content,
+ :plain, :entity, :error
+ ]
+
+ ATTR_NAME = /[\w.:-]+/
+ ATTR_VALUE_UNQUOTED = ATTR_NAME
+ TAG_END = /\/?>/
+ HEX = /[0-9a-fA-F]/
+ ENTITY = /
+ &
+ (?:
+ \w+
+ |
+ \#
+ (?:
+ \d+
+ |
+ x#{HEX}+
+ )
+ )
+ ;
+ /ox
+
+ PLAIN_STRING_CONTENT = {
+ "'" => /[^&'>\n]+/,
+ '"' => /[^&">\n]+/,
+ }
+
+ def reset
+ super
+ @state = :initial
+ end
+
+ private
+ def setup
+ @state = :initial
+ @plain_string_content = nil
+ end
+
+ def scan_tokens tokens, options
+
+ state = @state
+ plain_string_content = @plain_string_content
+
+ until eos?
+
+ kind = nil
+ match = nil
+
+ if scan(/\s+/m)
+ kind = :space
+
+ else
+
+ case state
+
+ when :initial
+ if scan(/<!--.*?-->/m)
+ kind = :comment
+ elsif scan(/<!DOCTYPE.*?>/m)
+ kind = :doctype
+ elsif scan(/<\?xml.*?\?>/m)
+ kind = :preprocessor
+ elsif scan(/<\?.*?\?>|<%.*?%>/m)
+ kind = :comment
+ elsif scan(/<\/[-\w.:]*>/m)
+ kind = :tag
+ elsif match = scan(/<[-\w.:]+>?/m)
+ kind = :tag
+ state = :attribute unless match[-1] == ?>
+ elsif scan(/[^<>&]+/)
+ kind = :plain
+ elsif scan(/#{ENTITY}/ox)
+ kind = :entity
+ elsif scan(/[<>&]/)
+ kind = :error
+ else
+ raise_inspect '[BUG] else-case reached with state %p' % [state], tokens
+ end
+
+ when :attribute
+ if scan(/#{TAG_END}/o)
+ kind = :tag
+ state = :initial
+ elsif scan(/#{ATTR_NAME}/o)
+ kind = :attribute_name
+ state = :attribute_equal
+ else
+ kind = :error
+ getch
+ end
+
+ when :attribute_equal
+ if scan(/=/)
+ kind = :operator
+ state = :attribute_value
+ elsif scan(/#{ATTR_NAME}/o)
+ kind = :attribute_name
+ elsif scan(/#{TAG_END}/o)
+ kind = :tag
+ state = :initial
+ elsif scan(/./)
+ kind = :error
+ state = :attribute
+ end
+
+ when :attribute_value
+ if scan(/#{ATTR_VALUE_UNQUOTED}/o)
+ kind = :attribute_value
+ state = :attribute
+ elsif match = scan(/["']/)
+ tokens << [:open, :string]
+ state = :attribute_value_string
+ plain_string_content = PLAIN_STRING_CONTENT[match]
+ kind = :delimiter
+ elsif scan(/#{TAG_END}/o)
+ kind = :tag
+ state = :initial
+ else
+ kind = :error
+ getch
+ end
+
+ when :attribute_value_string
+ if scan(plain_string_content)
+ kind = :content
+ elsif scan(/['"]/)
+ tokens << [matched, :delimiter]
+ tokens << [:close, :string]
+ state = :attribute
+ next
+ elsif scan(/#{ENTITY}/ox)
+ kind = :entity
+ elsif scan(/&/)
+ kind = :content
+ elsif scan(/[\n>]/)
+ tokens << [:close, :string]
+ kind = :error
+ state = :initial
+ end
+
+ else
+ raise_inspect 'Unknown state: %p' % [state], tokens
+
+ end
+
+ end
+
+ match ||= matched
+ if $CODERAY_DEBUG and not kind
+ raise_inspect 'Error token %p in line %d' %
+ [[match, kind], line], tokens, state
+ end
+ raise_inspect 'Empty token', tokens unless match
+
+ tokens << [match, kind]
+ end
+
+ if options[:keep_state]
+ @state = state
+ @plain_string_content = plain_string_content
+ end
+
+ tokens
+ end
+
+ end
+
+end
+end
--- /dev/null
+module CodeRay
+module Scanners
+
+ class Java < Scanner
+
+ include Streamable
+ register_for :java
+ helper :builtin_types
+
+ # http://java.sun.com/docs/books/tutorial/java/nutsandbolts/_keywords.html
+ KEYWORDS = %w[
+ assert break case catch continue default do else
+ finally for if instanceof import new package
+ return switch throw try typeof while
+ debugger export
+ ]
+ RESERVED = %w[ const goto ]
+ CONSTANTS = %w[ false null true ]
+ MAGIC_VARIABLES = %w[ this super ]
+ TYPES = %w[
+ boolean byte char class double enum float int interface long
+ short void
+ ] << '[]' # because int[] should be highlighted as a type
+ DIRECTIVES = %w[
+ abstract extends final implements native private protected public
+ static strictfp synchronized throws transient volatile
+ ]
+
+ IDENT_KIND = WordList.new(:ident).
+ add(KEYWORDS, :keyword).
+ add(RESERVED, :reserved).
+ add(CONSTANTS, :pre_constant).
+ add(MAGIC_VARIABLES, :local_variable).
+ add(TYPES, :type).
+ add(BuiltinTypes::List, :pre_type).
+ add(BuiltinTypes::List.select { |builtin| builtin[/(Error|Exception)$/] }, :exception).
+ add(DIRECTIVES, :directive)
+
+ ESCAPE = / [bfnrtv\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} /x
+ UNICODE_ESCAPE = / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} /x
+ STRING_CONTENT_PATTERN = {
+ "'" => /[^\\']+/,
+ '"' => /[^\\"]+/,
+ '/' => /[^\\\/]+/,
+ }
+ IDENT = /[a-zA-Z_][A-Za-z_0-9]*/
+
+ def scan_tokens tokens, options
+
+ state = :initial
+ string_delimiter = nil
+ import_clause = class_name_follows = last_token_dot = false
+
+ until eos?
+
+ kind = nil
+ match = nil
+
+ case state
+
+ when :initial
+
+ if match = scan(/ \s+ | \\\n /x)
+ tokens << [match, :space]
+ next
+
+ elsif match = scan(%r! // [^\n\\]* (?: \\. [^\n\\]* )* | /\* (?: .*? \*/ | .* ) !mx)
+ tokens << [match, :comment]
+ next
+
+ elsif import_clause && scan(/ #{IDENT} (?: \. #{IDENT} )* /ox)
+ kind = :include
+
+ elsif match = scan(/ #{IDENT} | \[\] /ox)
+ kind = IDENT_KIND[match]
+ if last_token_dot
+ kind = :ident
+ elsif class_name_follows
+ kind = :class
+ class_name_follows = false
+ else
+ import_clause = true if match == 'import'
+ class_name_follows = true if match == 'class' || match == 'interface'
+ end
+
+ elsif scan(/ \.(?!\d) | [,?:()\[\]}] | -- | \+\+ | && | \|\| | \*\*=? | [-+*\/%^~&|<>=!]=? | <<<?=? | >>>?=? /x)
+ kind = :operator
+
+ elsif scan(/;/)
+ import_clause = false
+ kind = :operator
+
+ elsif scan(/\{/)
+ class_name_follows = false
+ kind = :operator
+
+ elsif check(/[\d.]/)
+ if scan(/0[xX][0-9A-Fa-f]+/)
+ kind = :hex
+ elsif scan(/(?>0[0-7]+)(?![89.eEfF])/)
+ kind = :oct
+ elsif scan(/\d+[fFdD]|\d*\.\d+(?:[eE][+-]?\d+)?[fFdD]?|\d+[eE][+-]?\d+[fFdD]?/)
+ kind = :float
+ elsif scan(/\d+[lL]?/)
+ kind = :integer
+ end
+
+ elsif match = scan(/["']/)
+ tokens << [:open, :string]
+ state = :string
+ string_delimiter = match
+ kind = :delimiter
+
+ elsif scan(/ @ #{IDENT} /ox)
+ kind = :annotation
+
+ else
+ getch
+ kind = :error
+
+ end
+
+ when :string
+ if scan(STRING_CONTENT_PATTERN[string_delimiter])
+ kind = :content
+ elsif match = scan(/["'\/]/)
+ tokens << [match, :delimiter]
+ tokens << [:close, state]
+ string_delimiter = nil
+ state = :initial
+ next
+ elsif state == :string && (match = scan(/ \\ (?: #{ESCAPE} | #{UNICODE_ESCAPE} ) /mox))
+ if string_delimiter == "'" && !(match == "\\\\" || match == "\\'")
+ kind = :content
+ else
+ kind = :char
+ end
+ elsif scan(/\\./m)
+ kind = :content
+ elsif scan(/ \\ | $ /x)
+ tokens << [:close, state]
+ kind = :error
+ state = :initial
+ else
+ raise_inspect "else case \" reached; %p not handled." % peek(1), tokens
+ end
+
+ else
+ raise_inspect 'Unknown state', tokens
+
+ end
+
+ match ||= matched
+ if $CODERAY_DEBUG and not kind
+ raise_inspect 'Error token %p in line %d' %
+ [[match, kind], line], tokens
+ end
+ raise_inspect 'Empty token', tokens unless match
+
+ last_token_dot = match == '.'
+
+ tokens << [match, kind]
+
+ end
+
+ if state == :string
+ tokens << [:close, state]
+ end
+
+ tokens
+ end
+
+ end
+
+end
+end
--- /dev/null
+module CodeRay
+module Scanners
+
+ module Java::BuiltinTypes # :nodoc:
+
+ List = %w[
+ AbstractAction AbstractBorder AbstractButton AbstractCellEditor AbstractCollection
+ AbstractColorChooserPanel AbstractDocument AbstractExecutorService AbstractInterruptibleChannel
+ AbstractLayoutCache AbstractList AbstractListModel AbstractMap AbstractMethodError AbstractPreferences
+ AbstractQueue AbstractQueuedSynchronizer AbstractSelectableChannel AbstractSelectionKey AbstractSelector
+ AbstractSequentialList AbstractSet AbstractSpinnerModel AbstractTableModel AbstractUndoableEdit
+ AbstractWriter AccessControlContext AccessControlException AccessController AccessException Accessible
+ AccessibleAction AccessibleAttributeSequence AccessibleBundle AccessibleComponent AccessibleContext
+ AccessibleEditableText AccessibleExtendedComponent AccessibleExtendedTable AccessibleExtendedText
+ AccessibleHyperlink AccessibleHypertext AccessibleIcon AccessibleKeyBinding AccessibleObject
+ AccessibleRelation AccessibleRelationSet AccessibleResourceBundle AccessibleRole AccessibleSelection
+ AccessibleState AccessibleStateSet AccessibleStreamable AccessibleTable AccessibleTableModelChange
+ AccessibleText AccessibleTextSequence AccessibleValue AccountException AccountExpiredException
+ AccountLockedException AccountNotFoundException Acl AclEntry AclNotFoundException Action ActionEvent
+ ActionListener ActionMap ActionMapUIResource Activatable ActivateFailedException ActivationDesc
+ ActivationException ActivationGroup ActivationGroupDesc ActivationGroupID ActivationGroup_Stub
+ ActivationID ActivationInstantiator ActivationMonitor ActivationSystem Activator ActiveEvent
+ ActivityCompletedException ActivityRequiredException Adjustable AdjustmentEvent AdjustmentListener
+ Adler32 AffineTransform AffineTransformOp AlgorithmParameterGenerator AlgorithmParameterGeneratorSpi
+ AlgorithmParameters AlgorithmParameterSpec AlgorithmParametersSpi AllPermission AlphaComposite
+ AlreadyBoundException AlreadyConnectedException AncestorEvent AncestorListener AnnotatedElement
+ Annotation AnnotationFormatError AnnotationTypeMismatchException AppConfigurationEntry Appendable Applet
+ AppletContext AppletInitializer AppletStub Arc2D Area AreaAveragingScaleFilter ArithmeticException Array
+ ArrayBlockingQueue ArrayIndexOutOfBoundsException ArrayList Arrays ArrayStoreException ArrayType
+ AssertionError AsyncBoxView AsynchronousCloseException AtomicBoolean AtomicInteger AtomicIntegerArray
+ AtomicIntegerFieldUpdater AtomicLong AtomicLongArray AtomicLongFieldUpdater AtomicMarkableReference
+ AtomicReference AtomicReferenceArray AtomicReferenceFieldUpdater AtomicStampedReference Attribute
+ AttributeChangeNotification AttributeChangeNotificationFilter AttributedCharacterIterator
+ AttributedString AttributeException AttributeInUseException AttributeList AttributeModificationException
+ AttributeNotFoundException Attributes AttributeSet AttributeSetUtilities AttributeValueExp AudioClip
+ AudioFileFormat AudioFileReader AudioFileWriter AudioFormat AudioInputStream AudioPermission AudioSystem
+ AuthenticationException AuthenticationNotSupportedException Authenticator AuthorizeCallback
+ AuthPermission AuthProvider Autoscroll AWTError AWTEvent AWTEventListener AWTEventListenerProxy
+ AWTEventMulticaster AWTException AWTKeyStroke AWTPermission BackingStoreException
+ BadAttributeValueExpException BadBinaryOpValueExpException BadLocationException BadPaddingException
+ BadStringOperationException BandCombineOp BandedSampleModel BaseRowSet BasicArrowButton BasicAttribute
+ BasicAttributes BasicBorders BasicButtonListener BasicButtonUI BasicCheckBoxMenuItemUI BasicCheckBoxUI
+ BasicColorChooserUI BasicComboBoxEditor BasicComboBoxRenderer BasicComboBoxUI BasicComboPopup
+ BasicControl BasicDesktopIconUI BasicDesktopPaneUI BasicDirectoryModel BasicEditorPaneUI
+ BasicFileChooserUI BasicFormattedTextFieldUI BasicGraphicsUtils BasicHTML BasicIconFactory
+ BasicInternalFrameTitlePane BasicInternalFrameUI BasicLabelUI BasicListUI BasicLookAndFeel
+ BasicMenuBarUI BasicMenuItemUI BasicMenuUI BasicOptionPaneUI BasicPanelUI BasicPasswordFieldUI
+ BasicPermission BasicPopupMenuSeparatorUI BasicPopupMenuUI BasicProgressBarUI BasicRadioButtonMenuItemUI
+ BasicRadioButtonUI BasicRootPaneUI BasicScrollBarUI BasicScrollPaneUI BasicSeparatorUI BasicSliderUI
+ BasicSpinnerUI BasicSplitPaneDivider BasicSplitPaneUI BasicStroke BasicTabbedPaneUI BasicTableHeaderUI
+ BasicTableUI BasicTextAreaUI BasicTextFieldUI BasicTextPaneUI BasicTextUI BasicToggleButtonUI
+ BasicToolBarSeparatorUI BasicToolBarUI BasicToolTipUI BasicTreeUI BasicViewportUI BatchUpdateException
+ BeanContext BeanContextChild BeanContextChildComponentProxy BeanContextChildSupport
+ BeanContextContainerProxy BeanContextEvent BeanContextMembershipEvent BeanContextMembershipListener
+ BeanContextProxy BeanContextServiceAvailableEvent BeanContextServiceProvider
+ BeanContextServiceProviderBeanInfo BeanContextServiceRevokedEvent BeanContextServiceRevokedListener
+ BeanContextServices BeanContextServicesListener BeanContextServicesSupport BeanContextSupport
+ BeanDescriptor BeanInfo Beans BevelBorder Bidi BigDecimal BigInteger BinaryRefAddr BindException Binding
+ BitSet Blob BlockingQueue BlockView BMPImageWriteParam Book Boolean BooleanControl Border BorderFactory
+ BorderLayout BorderUIResource BoundedRangeModel Box BoxLayout BoxView BreakIterator
+ BrokenBarrierException Buffer BufferCapabilities BufferedImage BufferedImageFilter BufferedImageOp
+ BufferedInputStream BufferedOutputStream BufferedReader BufferedWriter BufferOverflowException
+ BufferStrategy BufferUnderflowException Button ButtonGroup ButtonModel ButtonUI Byte
+ ByteArrayInputStream ByteArrayOutputStream ByteBuffer ByteChannel ByteLookupTable ByteOrder CachedRowSet
+ CacheRequest CacheResponse Calendar Callable CallableStatement Callback CallbackHandler
+ CancelablePrintJob CancellationException CancelledKeyException CannotProceedException
+ CannotRedoException CannotUndoException Canvas CardLayout Caret CaretEvent CaretListener CellEditor
+ CellEditorListener CellRendererPane Certificate CertificateEncodingException CertificateException
+ CertificateExpiredException CertificateFactory CertificateFactorySpi CertificateNotYetValidException
+ CertificateParsingException CertPath CertPathBuilder CertPathBuilderException CertPathBuilderResult
+ CertPathBuilderSpi CertPathParameters CertPathTrustManagerParameters CertPathValidator
+ CertPathValidatorException CertPathValidatorResult CertPathValidatorSpi CertSelector CertStore
+ CertStoreException CertStoreParameters CertStoreSpi ChangedCharSetException ChangeEvent ChangeListener
+ Channel Channels Character CharacterCodingException CharacterIterator CharArrayReader CharArrayWriter
+ CharBuffer CharConversionException CharSequence Charset CharsetDecoder CharsetEncoder CharsetProvider
+ Checkbox CheckboxGroup CheckboxMenuItem CheckedInputStream CheckedOutputStream Checksum Choice
+ ChoiceCallback ChoiceFormat Chromaticity Cipher CipherInputStream CipherOutputStream CipherSpi Class
+ ClassCastException ClassCircularityError ClassDefinition ClassDesc ClassFileTransformer ClassFormatError
+ ClassLoader ClassLoaderRepository ClassLoadingMXBean ClassNotFoundException Clip Clipboard
+ ClipboardOwner Clob Cloneable CloneNotSupportedException Closeable ClosedByInterruptException
+ ClosedChannelException ClosedSelectorException CMMException CoderMalfunctionError CoderResult CodeSigner
+ CodeSource CodingErrorAction CollationElementIterator CollationKey Collator Collection
+ CollectionCertStoreParameters Collections Color ColorChooserComponentFactory ColorChooserUI
+ ColorConvertOp ColorModel ColorSelectionModel ColorSpace ColorSupported ColorType ColorUIResource
+ ComboBoxEditor ComboBoxModel ComboBoxUI ComboPopup CommunicationException Comparable Comparator
+ CompilationMXBean Compiler CompletionService Component ComponentAdapter ComponentColorModel
+ ComponentEvent ComponentInputMap ComponentInputMapUIResource ComponentListener ComponentOrientation
+ ComponentSampleModel ComponentUI ComponentView Composite CompositeContext CompositeData
+ CompositeDataSupport CompositeName CompositeType CompositeView CompoundBorder CompoundControl
+ CompoundEdit CompoundName Compression ConcurrentHashMap ConcurrentLinkedQueue ConcurrentMap
+ ConcurrentModificationException Condition Configuration ConfigurationException ConfirmationCallback
+ ConnectException ConnectIOException Connection ConnectionEvent ConnectionEventListener
+ ConnectionPendingException ConnectionPoolDataSource ConsoleHandler Constructor Container
+ ContainerAdapter ContainerEvent ContainerListener ContainerOrderFocusTraversalPolicy ContentHandler
+ ContentHandlerFactory ContentModel Context ContextNotEmptyException ContextualRenderedImageFactory
+ Control ControlFactory ControllerEventListener ConvolveOp CookieHandler Copies CopiesSupported
+ CopyOnWriteArrayList CopyOnWriteArraySet CountDownLatch CounterMonitor CounterMonitorMBean CRC32
+ CredentialException CredentialExpiredException CredentialNotFoundException CRL CRLException CRLSelector
+ CropImageFilter CSS CubicCurve2D Currency Cursor Customizer CyclicBarrier DatabaseMetaData DataBuffer
+ DataBufferByte DataBufferDouble DataBufferFloat DataBufferInt DataBufferShort DataBufferUShort
+ DataFlavor DataFormatException DatagramChannel DatagramPacket DatagramSocket DatagramSocketImpl
+ DatagramSocketImplFactory DataInput DataInputStream DataLine DataOutput DataOutputStream DataSource
+ DataTruncation DatatypeConfigurationException DatatypeConstants DatatypeFactory Date DateFormat
+ DateFormatSymbols DateFormatter DateTimeAtCompleted DateTimeAtCreation DateTimeAtProcessing
+ DateTimeSyntax DebugGraphics DecimalFormat DecimalFormatSymbols DefaultBoundedRangeModel
+ DefaultButtonModel DefaultCaret DefaultCellEditor DefaultColorSelectionModel DefaultComboBoxModel
+ DefaultDesktopManager DefaultEditorKit DefaultFocusManager DefaultFocusTraversalPolicy DefaultFormatter
+ DefaultFormatterFactory DefaultHighlighter DefaultKeyboardFocusManager DefaultListCellRenderer
+ DefaultListModel DefaultListSelectionModel DefaultLoaderRepository DefaultMenuLayout DefaultMetalTheme
+ DefaultMutableTreeNode DefaultPersistenceDelegate DefaultSingleSelectionModel DefaultStyledDocument
+ DefaultTableCellRenderer DefaultTableColumnModel DefaultTableModel DefaultTextUI DefaultTreeCellEditor
+ DefaultTreeCellRenderer DefaultTreeModel DefaultTreeSelectionModel Deflater DeflaterOutputStream Delayed
+ DelayQueue DelegationPermission Deprecated Descriptor DescriptorAccess DescriptorSupport DESedeKeySpec
+ DesignMode DESKeySpec DesktopIconUI DesktopManager DesktopPaneUI Destination Destroyable
+ DestroyFailedException DGC DHGenParameterSpec DHKey DHParameterSpec DHPrivateKey DHPrivateKeySpec
+ DHPublicKey DHPublicKeySpec Dialog Dictionary DigestException DigestInputStream DigestOutputStream
+ Dimension Dimension2D DimensionUIResource DirContext DirectColorModel DirectoryManager DirObjectFactory
+ DirStateFactory DisplayMode DnDConstants Doc DocAttribute DocAttributeSet DocFlavor DocPrintJob Document
+ DocumentBuilder DocumentBuilderFactory Documented DocumentEvent DocumentFilter DocumentListener
+ DocumentName DocumentParser DomainCombiner DOMLocator DOMResult DOMSource Double DoubleBuffer
+ DragGestureEvent DragGestureListener DragGestureRecognizer DragSource DragSourceAdapter
+ DragSourceContext DragSourceDragEvent DragSourceDropEvent DragSourceEvent DragSourceListener
+ DragSourceMotionListener Driver DriverManager DriverPropertyInfo DropTarget DropTargetAdapter
+ DropTargetContext DropTargetDragEvent DropTargetDropEvent DropTargetEvent DropTargetListener DSAKey
+ DSAKeyPairGenerator DSAParameterSpec DSAParams DSAPrivateKey DSAPrivateKeySpec DSAPublicKey
+ DSAPublicKeySpec DTD DTDConstants DuplicateFormatFlagsException Duration DynamicMBean ECField ECFieldF2m
+ ECFieldFp ECGenParameterSpec ECKey ECParameterSpec ECPoint ECPrivateKey ECPrivateKeySpec ECPublicKey
+ ECPublicKeySpec EditorKit Element ElementIterator ElementType Ellipse2D EllipticCurve EmptyBorder
+ EmptyStackException EncodedKeySpec Encoder EncryptedPrivateKeyInfo Entity Enum
+ EnumConstantNotPresentException EnumControl Enumeration EnumMap EnumSet EnumSyntax EOFException Error
+ ErrorListener ErrorManager EtchedBorder Event EventContext EventDirContext EventHandler EventListener
+ EventListenerList EventListenerProxy EventObject EventQueue EventSetDescriptor Exception
+ ExceptionInInitializerError ExceptionListener Exchanger ExecutionException Executor
+ ExecutorCompletionService Executors ExecutorService ExemptionMechanism ExemptionMechanismException
+ ExemptionMechanismSpi ExpandVetoException ExportException Expression ExtendedRequest ExtendedResponse
+ Externalizable FactoryConfigurationError FailedLoginException FeatureDescriptor Fidelity Field
+ FieldPosition FieldView File FileCacheImageInputStream FileCacheImageOutputStream FileChannel
+ FileChooserUI FileDescriptor FileDialog FileFilter FileHandler FileImageInputStream
+ FileImageOutputStream FileInputStream FileLock FileLockInterruptionException FilenameFilter FileNameMap
+ FileNotFoundException FileOutputStream FilePermission FileReader FileSystemView FileView FileWriter
+ Filter FilteredImageSource FilteredRowSet FilterInputStream FilterOutputStream FilterReader FilterWriter
+ Finishings FixedHeightLayoutCache FlatteningPathIterator FlavorEvent FlavorException FlavorListener
+ FlavorMap FlavorTable Float FloatBuffer FloatControl FlowLayout FlowView Flushable FocusAdapter
+ FocusEvent FocusListener FocusManager FocusTraversalPolicy Font FontFormatException FontMetrics
+ FontRenderContext FontUIResource Format FormatConversionProvider FormatFlagsConversionMismatchException
+ Formattable FormattableFlags Formatter FormatterClosedException FormSubmitEvent FormView Frame Future
+ FutureTask GapContent GarbageCollectorMXBean GatheringByteChannel GaugeMonitor GaugeMonitorMBean
+ GeneralPath GeneralSecurityException GenericArrayType GenericDeclaration GenericSignatureFormatError
+ GlyphJustificationInfo GlyphMetrics GlyphVector GlyphView GradientPaint GraphicAttribute Graphics
+ Graphics2D GraphicsConfigTemplate GraphicsConfiguration GraphicsDevice GraphicsEnvironment GrayFilter
+ GregorianCalendar GridBagConstraints GridBagLayout GridLayout Group Guard GuardedObject GZIPInputStream
+ GZIPOutputStream Handler HandshakeCompletedEvent HandshakeCompletedListener HasControls HashAttributeSet
+ HashDocAttributeSet HashMap HashPrintJobAttributeSet HashPrintRequestAttributeSet
+ HashPrintServiceAttributeSet HashSet Hashtable HeadlessException HierarchyBoundsAdapter
+ HierarchyBoundsListener HierarchyEvent HierarchyListener Highlighter HostnameVerifier HTML HTMLDocument
+ HTMLEditorKit HTMLFrameHyperlinkEvent HTMLWriter HttpRetryException HttpsURLConnection HttpURLConnection
+ HyperlinkEvent HyperlinkListener ICC_ColorSpace ICC_Profile ICC_ProfileGray ICC_ProfileRGB Icon
+ IconUIResource IconView Identity IdentityHashMap IdentityScope IIOByteBuffer IIOException IIOImage
+ IIOInvalidTreeException IIOMetadata IIOMetadataController IIOMetadataFormat IIOMetadataFormatImpl
+ IIOMetadataNode IIOParam IIOParamController IIOReadProgressListener IIOReadUpdateListener
+ IIOReadWarningListener IIORegistry IIOServiceProvider IIOWriteProgressListener IIOWriteWarningListener
+ IllegalAccessError IllegalAccessException IllegalArgumentException IllegalBlockingModeException
+ IllegalBlockSizeException IllegalCharsetNameException IllegalClassFormatException
+ IllegalComponentStateException IllegalFormatCodePointException IllegalFormatConversionException
+ IllegalFormatException IllegalFormatFlagsException IllegalFormatPrecisionException
+ IllegalFormatWidthException IllegalMonitorStateException IllegalPathStateException
+ IllegalSelectorException IllegalStateException IllegalThreadStateException Image ImageCapabilities
+ ImageConsumer ImageFilter ImageGraphicAttribute ImageIcon ImageInputStream ImageInputStreamImpl
+ ImageInputStreamSpi ImageIO ImageObserver ImageOutputStream ImageOutputStreamImpl ImageOutputStreamSpi
+ ImageProducer ImageReader ImageReaderSpi ImageReaderWriterSpi ImageReadParam ImageTranscoder
+ ImageTranscoderSpi ImageTypeSpecifier ImageView ImageWriteParam ImageWriter ImageWriterSpi
+ ImagingOpException IncompatibleClassChangeError IncompleteAnnotationException IndexColorModel
+ IndexedPropertyChangeEvent IndexedPropertyDescriptor IndexOutOfBoundsException Inet4Address Inet6Address
+ InetAddress InetSocketAddress Inflater InflaterInputStream InheritableThreadLocal Inherited
+ InitialContext InitialContextFactory InitialContextFactoryBuilder InitialDirContext InitialLdapContext
+ InlineView InputContext InputEvent InputMap InputMapUIResource InputMethod InputMethodContext
+ InputMethodDescriptor InputMethodEvent InputMethodHighlight InputMethodListener InputMethodRequests
+ InputMismatchException InputStream InputStreamReader InputSubset InputVerifier Insets InsetsUIResource
+ InstanceAlreadyExistsException InstanceNotFoundException InstantiationError InstantiationException
+ Instrument Instrumentation InsufficientResourcesException IntBuffer Integer IntegerSyntax InternalError
+ InternalFrameAdapter InternalFrameEvent InternalFrameFocusTraversalPolicy InternalFrameListener
+ InternalFrameUI InternationalFormatter InterruptedException InterruptedIOException
+ InterruptedNamingException InterruptibleChannel IntrospectionException Introspector
+ InvalidActivityException InvalidAlgorithmParameterException InvalidApplicationException
+ InvalidAttributeIdentifierException InvalidAttributesException InvalidAttributeValueException
+ InvalidClassException InvalidDnDOperationException InvalidKeyException InvalidKeySpecException
+ InvalidMarkException InvalidMidiDataException InvalidNameException InvalidObjectException
+ InvalidOpenTypeException InvalidParameterException InvalidParameterSpecException
+ InvalidPreferencesFormatException InvalidPropertiesFormatException InvalidRelationIdException
+ InvalidRelationServiceException InvalidRelationTypeException InvalidRoleInfoException
+ InvalidRoleValueException InvalidSearchControlsException InvalidSearchFilterException
+ InvalidTargetObjectTypeException InvalidTransactionException InvocationEvent InvocationHandler
+ InvocationTargetException IOException ItemEvent ItemListener ItemSelectable Iterable Iterator
+ IvParameterSpec JApplet JarEntry JarException JarFile JarInputStream JarOutputStream JarURLConnection
+ JButton JCheckBox JCheckBoxMenuItem JColorChooser JComboBox JComponent JdbcRowSet JDesktopPane JDialog
+ JEditorPane JFileChooser JFormattedTextField JFrame JInternalFrame JLabel JLayeredPane JList JMenu
+ JMenuBar JMenuItem JMException JMRuntimeException JMXAuthenticator JMXConnectionNotification
+ JMXConnector JMXConnectorFactory JMXConnectorProvider JMXConnectorServer JMXConnectorServerFactory
+ JMXConnectorServerMBean JMXConnectorServerProvider JMXPrincipal JMXProviderException
+ JMXServerErrorException JMXServiceURL JobAttributes JobHoldUntil JobImpressions JobImpressionsCompleted
+ JobImpressionsSupported JobKOctets JobKOctetsProcessed JobKOctetsSupported JobMediaSheets
+ JobMediaSheetsCompleted JobMediaSheetsSupported JobMessageFromOperator JobName JobOriginatingUserName
+ JobPriority JobPrioritySupported JobSheets JobState JobStateReason JobStateReasons Joinable JoinRowSet
+ JOptionPane JPanel JPasswordField JPEGHuffmanTable JPEGImageReadParam JPEGImageWriteParam JPEGQTable
+ JPopupMenu JProgressBar JRadioButton JRadioButtonMenuItem JRootPane JScrollBar JScrollPane JSeparator
+ JSlider JSpinner JSplitPane JTabbedPane JTable JTableHeader JTextArea JTextComponent JTextField
+ JTextPane JToggleButton JToolBar JToolTip JTree JViewport JWindow KerberosKey KerberosPrincipal
+ KerberosTicket Kernel Key KeyAdapter KeyAgreement KeyAgreementSpi KeyAlreadyExistsException
+ KeyboardFocusManager KeyEvent KeyEventDispatcher KeyEventPostProcessor KeyException KeyFactory
+ KeyFactorySpi KeyGenerator KeyGeneratorSpi KeyListener KeyManagementException KeyManager
+ KeyManagerFactory KeyManagerFactorySpi Keymap KeyPair KeyPairGenerator KeyPairGeneratorSpi KeyRep
+ KeySpec KeyStore KeyStoreBuilderParameters KeyStoreException KeyStoreSpi KeyStroke Label LabelUI
+ LabelView LanguageCallback LastOwnerException LayeredHighlighter LayoutFocusTraversalPolicy
+ LayoutManager LayoutManager2 LayoutQueue LDAPCertStoreParameters LdapContext LdapName
+ LdapReferralException Lease Level LimitExceededException Line Line2D LineBorder LineBreakMeasurer
+ LineEvent LineListener LineMetrics LineNumberInputStream LineNumberReader LineUnavailableException
+ LinkageError LinkedBlockingQueue LinkedHashMap LinkedHashSet LinkedList LinkException LinkLoopException
+ LinkRef List ListCellRenderer ListDataEvent ListDataListener ListenerNotFoundException ListIterator
+ ListModel ListResourceBundle ListSelectionEvent ListSelectionListener ListSelectionModel ListUI ListView
+ LoaderHandler Locale LocateRegistry Lock LockSupport Logger LoggingMXBean LoggingPermission LoginContext
+ LoginException LoginModule LogManager LogRecord LogStream Long LongBuffer LookAndFeel LookupOp
+ LookupTable Mac MacSpi MalformedInputException MalformedLinkException MalformedObjectNameException
+ MalformedParameterizedTypeException MalformedURLException ManagementFactory ManagementPermission
+ ManageReferralControl ManagerFactoryParameters Manifest Map MappedByteBuffer MarshalException
+ MarshalledObject MaskFormatter Matcher MatchResult Math MathContext MatteBorder MBeanAttributeInfo
+ MBeanConstructorInfo MBeanException MBeanFeatureInfo MBeanInfo MBeanNotificationInfo MBeanOperationInfo
+ MBeanParameterInfo MBeanPermission MBeanRegistration MBeanRegistrationException MBeanServer
+ MBeanServerBuilder MBeanServerConnection MBeanServerDelegate MBeanServerDelegateMBean MBeanServerFactory
+ MBeanServerForwarder MBeanServerInvocationHandler MBeanServerNotification MBeanServerNotificationFilter
+ MBeanServerPermission MBeanTrustPermission Media MediaName MediaPrintableArea MediaSize MediaSizeName
+ MediaTracker MediaTray Member MemoryCacheImageInputStream MemoryCacheImageOutputStream MemoryHandler
+ MemoryImageSource MemoryManagerMXBean MemoryMXBean MemoryNotificationInfo MemoryPoolMXBean MemoryType
+ MemoryUsage Menu MenuBar MenuBarUI MenuComponent MenuContainer MenuDragMouseEvent MenuDragMouseListener
+ MenuElement MenuEvent MenuItem MenuItemUI MenuKeyEvent MenuKeyListener MenuListener MenuSelectionManager
+ MenuShortcut MessageDigest MessageDigestSpi MessageFormat MetaEventListener MetalBorders MetalButtonUI
+ MetalCheckBoxIcon MetalCheckBoxUI MetalComboBoxButton MetalComboBoxEditor MetalComboBoxIcon
+ MetalComboBoxUI MetalDesktopIconUI MetalFileChooserUI MetalIconFactory MetalInternalFrameTitlePane
+ MetalInternalFrameUI MetalLabelUI MetalLookAndFeel MetalMenuBarUI MetalPopupMenuSeparatorUI
+ MetalProgressBarUI MetalRadioButtonUI MetalRootPaneUI MetalScrollBarUI MetalScrollButton
+ MetalScrollPaneUI MetalSeparatorUI MetalSliderUI MetalSplitPaneUI MetalTabbedPaneUI MetalTextFieldUI
+ MetalTheme MetalToggleButtonUI MetalToolBarUI MetalToolTipUI MetalTreeUI MetaMessage Method
+ MethodDescriptor MGF1ParameterSpec MidiChannel MidiDevice MidiDeviceProvider MidiEvent MidiFileFormat
+ MidiFileReader MidiFileWriter MidiMessage MidiSystem MidiUnavailableException MimeTypeParseException
+ MinimalHTMLWriter MissingFormatArgumentException MissingFormatWidthException MissingResourceException
+ Mixer MixerProvider MLet MLetMBean ModelMBean ModelMBeanAttributeInfo ModelMBeanConstructorInfo
+ ModelMBeanInfo ModelMBeanInfoSupport ModelMBeanNotificationBroadcaster ModelMBeanNotificationInfo
+ ModelMBeanOperationInfo ModificationItem Modifier Monitor MonitorMBean MonitorNotification
+ MonitorSettingException MouseAdapter MouseDragGestureRecognizer MouseEvent MouseInfo MouseInputAdapter
+ MouseInputListener MouseListener MouseMotionAdapter MouseMotionListener MouseWheelEvent
+ MouseWheelListener MultiButtonUI MulticastSocket MultiColorChooserUI MultiComboBoxUI MultiDesktopIconUI
+ MultiDesktopPaneUI MultiDoc MultiDocPrintJob MultiDocPrintService MultiFileChooserUI
+ MultiInternalFrameUI MultiLabelUI MultiListUI MultiLookAndFeel MultiMenuBarUI MultiMenuItemUI
+ MultiOptionPaneUI MultiPanelUI MultiPixelPackedSampleModel MultipleDocumentHandling MultipleMaster
+ MultiPopupMenuUI MultiProgressBarUI MultiRootPaneUI MultiScrollBarUI MultiScrollPaneUI MultiSeparatorUI
+ MultiSliderUI MultiSpinnerUI MultiSplitPaneUI MultiTabbedPaneUI MultiTableHeaderUI MultiTableUI
+ MultiTextUI MultiToolBarUI MultiToolTipUI MultiTreeUI MultiViewportUI MutableAttributeSet
+ MutableComboBoxModel MutableTreeNode Name NameAlreadyBoundException NameCallback NameClassPair
+ NameNotFoundException NameParser NamespaceChangeListener NamespaceContext Naming NamingEnumeration
+ NamingEvent NamingException NamingExceptionEvent NamingListener NamingManager NamingSecurityException
+ NavigationFilter NegativeArraySizeException NetPermission NetworkInterface NoClassDefFoundError
+ NoConnectionPendingException NodeChangeEvent NodeChangeListener NoInitialContextException
+ NoninvertibleTransformException NonReadableChannelException NonWritableChannelException
+ NoPermissionException NoRouteToHostException NoSuchAlgorithmException NoSuchAttributeException
+ NoSuchElementException NoSuchFieldError NoSuchFieldException NoSuchMethodError NoSuchMethodException
+ NoSuchObjectException NoSuchPaddingException NoSuchProviderException NotActiveException
+ NotBoundException NotCompliantMBeanException NotContextException Notification NotificationBroadcaster
+ NotificationBroadcasterSupport NotificationEmitter NotificationFilter NotificationFilterSupport
+ NotificationListener NotificationResult NotOwnerException NotSerializableException NotYetBoundException
+ NotYetConnectedException NullCipher NullPointerException Number NumberFormat NumberFormatException
+ NumberFormatter NumberOfDocuments NumberOfInterveningJobs NumberUp NumberUpSupported NumericShaper
+ OAEPParameterSpec Object ObjectChangeListener ObjectFactory ObjectFactoryBuilder ObjectInput
+ ObjectInputStream ObjectInputValidation ObjectInstance ObjectName ObjectOutput ObjectOutputStream
+ ObjectStreamClass ObjectStreamConstants ObjectStreamException ObjectStreamField ObjectView ObjID
+ Observable Observer OceanTheme OpenDataException OpenMBeanAttributeInfo OpenMBeanAttributeInfoSupport
+ OpenMBeanConstructorInfo OpenMBeanConstructorInfoSupport OpenMBeanInfo OpenMBeanInfoSupport
+ OpenMBeanOperationInfo OpenMBeanOperationInfoSupport OpenMBeanParameterInfo
+ OpenMBeanParameterInfoSupport OpenType OperatingSystemMXBean Operation OperationNotSupportedException
+ OperationsException Option OptionalDataException OptionPaneUI OrientationRequested OutOfMemoryError
+ OutputDeviceAssigned OutputKeys OutputStream OutputStreamWriter OverlappingFileLockException
+ OverlayLayout Override Owner Pack200 Package PackedColorModel Pageable PageAttributes
+ PagedResultsControl PagedResultsResponseControl PageFormat PageRanges PagesPerMinute PagesPerMinuteColor
+ Paint PaintContext PaintEvent Panel PanelUI Paper ParagraphView ParameterBlock ParameterDescriptor
+ ParameterizedType ParameterMetaData ParseException ParsePosition Parser ParserConfigurationException
+ ParserDelegator PartialResultException PasswordAuthentication PasswordCallback PasswordView Patch
+ PathIterator Pattern PatternSyntaxException PBEKey PBEKeySpec PBEParameterSpec PDLOverrideSupported
+ Permission PermissionCollection Permissions PersistenceDelegate PersistentMBean PhantomReference Pipe
+ PipedInputStream PipedOutputStream PipedReader PipedWriter PixelGrabber PixelInterleavedSampleModel
+ PKCS8EncodedKeySpec PKIXBuilderParameters PKIXCertPathBuilderResult PKIXCertPathChecker
+ PKIXCertPathValidatorResult PKIXParameters PlainDocument PlainView Point Point2D PointerInfo Policy
+ PolicyNode PolicyQualifierInfo Polygon PooledConnection Popup PopupFactory PopupMenu PopupMenuEvent
+ PopupMenuListener PopupMenuUI Port PortableRemoteObject PortableRemoteObjectDelegate
+ PortUnreachableException Position Predicate PreferenceChangeEvent PreferenceChangeListener Preferences
+ PreferencesFactory PreparedStatement PresentationDirection Principal Printable PrinterAbortException
+ PrinterException PrinterGraphics PrinterInfo PrinterIOException PrinterIsAcceptingJobs PrinterJob
+ PrinterLocation PrinterMakeAndModel PrinterMessageFromOperator PrinterMoreInfo
+ PrinterMoreInfoManufacturer PrinterName PrinterResolution PrinterState PrinterStateReason
+ PrinterStateReasons PrinterURI PrintEvent PrintException PrintGraphics PrintJob PrintJobAdapter
+ PrintJobAttribute PrintJobAttributeEvent PrintJobAttributeListener PrintJobAttributeSet PrintJobEvent
+ PrintJobListener PrintQuality PrintRequestAttribute PrintRequestAttributeSet PrintService
+ PrintServiceAttribute PrintServiceAttributeEvent PrintServiceAttributeListener PrintServiceAttributeSet
+ PrintServiceLookup PrintStream PrintWriter PriorityBlockingQueue PriorityQueue PrivateClassLoader
+ PrivateCredentialPermission PrivateKey PrivateMLet PrivilegedAction PrivilegedActionException
+ PrivilegedExceptionAction Process ProcessBuilder ProfileDataException ProgressBarUI ProgressMonitor
+ ProgressMonitorInputStream Properties PropertyChangeEvent PropertyChangeListener
+ PropertyChangeListenerProxy PropertyChangeSupport PropertyDescriptor PropertyEditor
+ PropertyEditorManager PropertyEditorSupport PropertyPermission PropertyResourceBundle
+ PropertyVetoException ProtectionDomain ProtocolException Provider ProviderException Proxy ProxySelector
+ PSource PSSParameterSpec PublicKey PushbackInputStream PushbackReader QName QuadCurve2D Query QueryEval
+ QueryExp Queue QueuedJobCount Random RandomAccess RandomAccessFile Raster RasterFormatException RasterOp
+ RC2ParameterSpec RC5ParameterSpec Rdn Readable ReadableByteChannel Reader ReadOnlyBufferException
+ ReadWriteLock RealmCallback RealmChoiceCallback Receiver Rectangle Rectangle2D RectangularShape
+ ReentrantLock ReentrantReadWriteLock Ref RefAddr Reference Referenceable ReferenceQueue
+ ReferenceUriSchemesSupported ReferralException ReflectionException ReflectPermission Refreshable
+ RefreshFailedException Region RegisterableService Registry RegistryHandler RejectedExecutionException
+ RejectedExecutionHandler Relation RelationException RelationNotFoundException RelationNotification
+ RelationService RelationServiceMBean RelationServiceNotRegisteredException RelationSupport
+ RelationSupportMBean RelationType RelationTypeNotFoundException RelationTypeSupport Remote RemoteCall
+ RemoteException RemoteObject RemoteObjectInvocationHandler RemoteRef RemoteServer RemoteStub
+ RenderableImage RenderableImageOp RenderableImageProducer RenderContext RenderedImage
+ RenderedImageFactory Renderer RenderingHints RepaintManager ReplicateScaleFilter RequestingUserName
+ RequiredModelMBean RescaleOp ResolutionSyntax Resolver ResolveResult ResourceBundle ResponseCache Result
+ ResultSet ResultSetMetaData Retention RetentionPolicy ReverbType RGBImageFilter RMIClassLoader
+ RMIClassLoaderSpi RMIClientSocketFactory RMIConnection RMIConnectionImpl RMIConnectionImpl_Stub
+ RMIConnector RMIConnectorServer RMIFailureHandler RMIIIOPServerImpl RMIJRMPServerImpl
+ RMISecurityException RMISecurityManager RMIServer RMIServerImpl RMIServerImpl_Stub
+ RMIServerSocketFactory RMISocketFactory Robot Role RoleInfo RoleInfoNotFoundException RoleList
+ RoleNotFoundException RoleResult RoleStatus RoleUnresolved RoleUnresolvedList RootPaneContainer
+ RootPaneUI RoundingMode RoundRectangle2D RowMapper RowSet RowSetEvent RowSetInternal RowSetListener
+ RowSetMetaData RowSetMetaDataImpl RowSetReader RowSetWarning RowSetWriter RSAKey RSAKeyGenParameterSpec
+ RSAMultiPrimePrivateCrtKey RSAMultiPrimePrivateCrtKeySpec RSAOtherPrimeInfo RSAPrivateCrtKey
+ RSAPrivateCrtKeySpec RSAPrivateKey RSAPrivateKeySpec RSAPublicKey RSAPublicKeySpec RTFEditorKit
+ RuleBasedCollator Runnable Runtime RuntimeErrorException RuntimeException RuntimeMBeanException
+ RuntimeMXBean RuntimeOperationsException RuntimePermission SampleModel Sasl SaslClient SaslClientFactory
+ SaslException SaslServer SaslServerFactory Savepoint SAXParser SAXParserFactory SAXResult SAXSource
+ SAXTransformerFactory Scanner ScatteringByteChannel ScheduledExecutorService ScheduledFuture
+ ScheduledThreadPoolExecutor Schema SchemaFactory SchemaFactoryLoader SchemaViolationException Scrollable
+ Scrollbar ScrollBarUI ScrollPane ScrollPaneAdjustable ScrollPaneConstants ScrollPaneLayout ScrollPaneUI
+ SealedObject SearchControls SearchResult SecretKey SecretKeyFactory SecretKeyFactorySpi SecretKeySpec
+ SecureCacheResponse SecureClassLoader SecureRandom SecureRandomSpi Security SecurityException
+ SecurityManager SecurityPermission Segment SelectableChannel SelectionKey Selector SelectorProvider
+ Semaphore SeparatorUI Sequence SequenceInputStream Sequencer SerialArray SerialBlob SerialClob
+ SerialDatalink SerialException Serializable SerializablePermission SerialJavaObject SerialRef
+ SerialStruct ServerCloneException ServerError ServerException ServerNotActiveException ServerRef
+ ServerRuntimeException ServerSocket ServerSocketChannel ServerSocketFactory ServiceNotFoundException
+ ServicePermission ServiceRegistry ServiceUI ServiceUIFactory ServiceUnavailableException Set
+ SetOfIntegerSyntax Severity Shape ShapeGraphicAttribute SheetCollate Short ShortBuffer
+ ShortBufferException ShortLookupTable ShortMessage Sides Signature SignatureException SignatureSpi
+ SignedObject Signer SimpleAttributeSet SimpleBeanInfo SimpleDateFormat SimpleDoc SimpleFormatter
+ SimpleTimeZone SimpleType SinglePixelPackedSampleModel SingleSelectionModel Size2DSyntax
+ SizeLimitExceededException SizeRequirements SizeSequence Skeleton SkeletonMismatchException
+ SkeletonNotFoundException SliderUI Socket SocketAddress SocketChannel SocketException SocketFactory
+ SocketHandler SocketImpl SocketImplFactory SocketOptions SocketPermission SocketSecurityException
+ SocketTimeoutException SoftBevelBorder SoftReference SortControl SortedMap SortedSet
+ SortingFocusTraversalPolicy SortKey SortResponseControl Soundbank SoundbankReader SoundbankResource
+ Source SourceDataLine SourceLocator SpinnerDateModel SpinnerListModel SpinnerModel SpinnerNumberModel
+ SpinnerUI SplitPaneUI Spring SpringLayout SQLData SQLException SQLInput SQLInputImpl SQLOutput
+ SQLOutputImpl SQLPermission SQLWarning SSLContext SSLContextSpi SSLEngine SSLEngineResult SSLException
+ SSLHandshakeException SSLKeyException SSLPeerUnverifiedException SSLPermission SSLProtocolException
+ SslRMIClientSocketFactory SslRMIServerSocketFactory SSLServerSocket SSLServerSocketFactory SSLSession
+ SSLSessionBindingEvent SSLSessionBindingListener SSLSessionContext SSLSocket SSLSocketFactory Stack
+ StackOverflowError StackTraceElement StandardMBean StartTlsRequest StartTlsResponse StateEdit
+ StateEditable StateFactory Statement StreamCorruptedException StreamHandler StreamPrintService
+ StreamPrintServiceFactory StreamResult StreamSource StreamTokenizer StrictMath String StringBuffer
+ StringBufferInputStream StringBuilder StringCharacterIterator StringContent
+ StringIndexOutOfBoundsException StringMonitor StringMonitorMBean StringReader StringRefAddr
+ StringSelection StringTokenizer StringValueExp StringWriter Stroke Struct Stub StubDelegate
+ StubNotFoundException Style StyleConstants StyleContext StyledDocument StyledEditorKit StyleSheet
+ Subject SubjectDelegationPermission SubjectDomainCombiner SupportedValuesAttribute SuppressWarnings
+ SwingConstants SwingPropertyChangeSupport SwingUtilities SyncFactory SyncFactoryException
+ SyncFailedException SynchronousQueue SyncProvider SyncProviderException SyncResolver SynthConstants
+ SynthContext Synthesizer SynthGraphicsUtils SynthLookAndFeel SynthPainter SynthStyle SynthStyleFactory
+ SysexMessage System SystemColor SystemFlavorMap TabableView TabbedPaneUI TabExpander TableCellEditor
+ TableCellRenderer TableColumn TableColumnModel TableColumnModelEvent TableColumnModelListener
+ TableHeaderUI TableModel TableModelEvent TableModelListener TableUI TableView TabSet TabStop TabularData
+ TabularDataSupport TabularType TagElement Target TargetDataLine TargetedNotification Templates
+ TemplatesHandler TextAction TextArea TextAttribute TextComponent TextEvent TextField TextHitInfo
+ TextInputCallback TextLayout TextListener TextMeasurer TextOutputCallback TextSyntax TextUI TexturePaint
+ Thread ThreadDeath ThreadFactory ThreadGroup ThreadInfo ThreadLocal ThreadMXBean ThreadPoolExecutor
+ Throwable Tie TileObserver Time TimeLimitExceededException TimeoutException Timer
+ TimerAlarmClockNotification TimerMBean TimerNotification TimerTask Timestamp TimeUnit TimeZone
+ TitledBorder ToolBarUI Toolkit ToolTipManager ToolTipUI TooManyListenersException Track
+ TransactionalWriter TransactionRequiredException TransactionRolledbackException Transferable
+ TransferHandler TransformAttribute Transformer TransformerConfigurationException TransformerException
+ TransformerFactory TransformerFactoryConfigurationError TransformerHandler Transmitter Transparency
+ TreeCellEditor TreeCellRenderer TreeExpansionEvent TreeExpansionListener TreeMap TreeModel
+ TreeModelEvent TreeModelListener TreeNode TreePath TreeSelectionEvent TreeSelectionListener
+ TreeSelectionModel TreeSet TreeUI TreeWillExpandListener TrustAnchor TrustManager TrustManagerFactory
+ TrustManagerFactorySpi Type TypeInfoProvider TypeNotPresentException Types TypeVariable UID UIDefaults
+ UIManager UIResource UndeclaredThrowableException UndoableEdit UndoableEditEvent UndoableEditListener
+ UndoableEditSupport UndoManager UnexpectedException UnicastRemoteObject UnknownError
+ UnknownFormatConversionException UnknownFormatFlagsException UnknownGroupException UnknownHostException
+ UnknownObjectException UnknownServiceException UnmappableCharacterException UnmarshalException
+ UnmodifiableClassException UnmodifiableSetException UnrecoverableEntryException
+ UnrecoverableKeyException Unreferenced UnresolvedAddressException UnresolvedPermission
+ UnsatisfiedLinkError UnsolicitedNotification UnsolicitedNotificationEvent
+ UnsolicitedNotificationListener UnsupportedAddressTypeException UnsupportedAudioFileException
+ UnsupportedCallbackException UnsupportedCharsetException UnsupportedClassVersionError
+ UnsupportedEncodingException UnsupportedFlavorException UnsupportedLookAndFeelException
+ UnsupportedOperationException URI URIException URIResolver URISyntax URISyntaxException URL
+ URLClassLoader URLConnection URLDecoder URLEncoder URLStreamHandler URLStreamHandlerFactory
+ UTFDataFormatException Util UtilDelegate Utilities UUID Validator ValidatorHandler ValueExp ValueHandler
+ ValueHandlerMultiFormat VariableHeightLayoutCache Vector VerifyError VetoableChangeListener
+ VetoableChangeListenerProxy VetoableChangeSupport View ViewFactory ViewportLayout ViewportUI
+ VirtualMachineError Visibility VMID VoiceStatus Void VolatileImage WeakHashMap WeakReference WebRowSet
+ WildcardType Window WindowAdapter WindowConstants WindowEvent WindowFocusListener WindowListener
+ WindowStateListener WrappedPlainView WritableByteChannel WritableRaster WritableRenderedImage
+ WriteAbortedException Writer X500Principal X500PrivateCredential X509Certificate X509CertSelector
+ X509CRL X509CRLEntry X509CRLSelector X509EncodedKeySpec X509ExtendedKeyManager X509Extension
+ X509KeyManager X509TrustManager XAConnection XADataSource XAException XAResource Xid XMLConstants
+ XMLDecoder XMLEncoder XMLFormatter XMLGregorianCalendar XMLParseException XmlReader XmlWriter XPath
+ XPathConstants XPathException XPathExpression XPathExpressionException XPathFactory
+ XPathFactoryConfigurationException XPathFunction XPathFunctionException XPathFunctionResolver
+ XPathVariableResolver ZipEntry ZipException ZipFile ZipInputStream ZipOutputStream ZoneView
+ ]
+
+ end
+
+end
+end
--- /dev/null
+module CodeRay
+module Scanners
+
+ class JavaScript < Scanner
+
+ include Streamable
+
+ register_for :java_script
+ file_extension 'js'
+
+ # The actual JavaScript keywords.
+ KEYWORDS = %w[
+ break case catch continue default delete do else
+ finally for function if in instanceof new
+ return switch throw try typeof var void while with
+ ]
+ PREDEFINED_CONSTANTS = %w[
+ false null true undefined
+ ]
+
+ MAGIC_VARIABLES = %w[ this arguments ] # arguments was introduced in JavaScript 1.4
+
+ KEYWORDS_EXPECTING_VALUE = WordList.new.add %w[
+ case delete in instanceof new return throw typeof with
+ ]
+
+ # Reserved for future use.
+ RESERVED_WORDS = %w[
+ abstract boolean byte char class debugger double enum export extends
+ final float goto implements import int interface long native package
+ private protected public short static super synchronized throws transient
+ volatile
+ ]
+
+ IDENT_KIND = WordList.new(:ident).
+ add(RESERVED_WORDS, :reserved).
+ add(PREDEFINED_CONSTANTS, :pre_constant).
+ add(MAGIC_VARIABLES, :local_variable).
+ add(KEYWORDS, :keyword)
+
+ ESCAPE = / [bfnrtv\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} /x
+ UNICODE_ESCAPE = / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} /x
+ REGEXP_ESCAPE = / [bBdDsSwW] /x
+ STRING_CONTENT_PATTERN = {
+ "'" => /[^\\']+/,
+ '"' => /[^\\"]+/,
+ '/' => /[^\\\/]+/,
+ }
+ KEY_CHECK_PATTERN = {
+ "'" => / [^\\']* (?: \\.? [^\\']* )* '? \s* : /x,
+ '"' => / [^\\"]* (?: \\.? [^\\"]* )* "? \s* : /x,
+ }
+
+ def scan_tokens tokens, options
+
+ state = :initial
+ string_delimiter = nil
+ value_expected = true
+ key_expected = false
+ function_expected = false
+
+ until eos?
+
+ kind = nil
+ match = nil
+
+ case state
+
+ when :initial
+
+ if match = scan(/ \s+ | \\\n /x)
+ value_expected = true if !value_expected && match.index(?\n)
+ tokens << [match, :space]
+ next
+
+ elsif scan(%r! // [^\n\\]* (?: \\. [^\n\\]* )* | /\* (?: .*? \*/ | .* ) !mx)
+ value_expected = true
+ kind = :comment
+
+ elsif check(/\.?\d/)
+ key_expected = value_expected = false
+ if scan(/0[xX][0-9A-Fa-f]+/)
+ kind = :hex
+ elsif scan(/(?>0[0-7]+)(?![89.eEfF])/)
+ kind = :oct
+ elsif scan(/\d+[fF]|\d*\.\d+(?:[eE][+-]?\d+)?[fF]?|\d+[eE][+-]?\d+[fF]?/)
+ kind = :float
+ elsif scan(/\d+/)
+ kind = :integer
+ end
+
+ elsif value_expected && match = scan(/<([[:alpha:]]\w*) (?: [^\/>]*\/> | .*?<\/\1>)/xim)
+ # FIXME: scan over nested tags
+ xml_scanner.tokenize match
+ value_expected = false
+ next
+
+ elsif match = scan(/ [-+*=<>?:;,!&^|(\[{~%]+ | \.(?!\d) /x)
+ value_expected = true
+ last_operator = match[-1]
+ key_expected = (last_operator == ?{) || (last_operator == ?,)
+ function_expected = false
+ kind = :operator
+
+ elsif scan(/ [)\]}]+ /x)
+ function_expected = key_expected = value_expected = false
+ kind = :operator
+
+ elsif match = scan(/ [$a-zA-Z_][A-Za-z_0-9$]* /x)
+ kind = IDENT_KIND[match]
+ value_expected = (kind == :keyword) && KEYWORDS_EXPECTING_VALUE[match]
+ # TODO: labels
+ if kind == :ident
+ if match.index(?$) # $ allowed inside an identifier
+ kind = :predefined
+ elsif function_expected
+ kind = :function
+ elsif check(/\s*[=:]\s*function\b/)
+ kind = :function
+ elsif key_expected && check(/\s*:/)
+ kind = :key
+ end
+ end
+ function_expected = (kind == :keyword) && (match == 'function')
+ key_expected = false
+
+ elsif match = scan(/["']/)
+ if key_expected && check(KEY_CHECK_PATTERN[match])
+ state = :key
+ else
+ state = :string
+ end
+ tokens << [:open, state]
+ string_delimiter = match
+ kind = :delimiter
+
+ elsif value_expected && (match = scan(/\/(?=\S)/))
+ tokens << [:open, :regexp]
+ state = :regexp
+ string_delimiter = '/'
+ kind = :delimiter
+
+ elsif scan(/ \/ /x)
+ value_expected = true
+ key_expected = false
+ kind = :operator
+
+ else
+ getch
+ kind = :error
+
+ end
+
+ when :string, :regexp, :key
+ if scan(STRING_CONTENT_PATTERN[string_delimiter])
+ kind = :content
+ elsif match = scan(/["'\/]/)
+ tokens << [match, :delimiter]
+ if state == :regexp
+ modifiers = scan(/[gim]+/)
+ tokens << [modifiers, :modifier] if modifiers && !modifiers.empty?
+ end
+ tokens << [:close, state]
+ string_delimiter = nil
+ key_expected = value_expected = false
+ state = :initial
+ next
+ elsif state != :regexp && (match = scan(/ \\ (?: #{ESCAPE} | #{UNICODE_ESCAPE} ) /mox))
+ if string_delimiter == "'" && !(match == "\\\\" || match == "\\'")
+ kind = :content
+ else
+ kind = :char
+ end
+ elsif state == :regexp && scan(/ \\ (?: #{ESCAPE} | #{REGEXP_ESCAPE} | #{UNICODE_ESCAPE} ) /mox)
+ kind = :char
+ elsif scan(/\\./m)
+ kind = :content
+ elsif scan(/ \\ | $ /x)
+ tokens << [:close, state]
+ kind = :error
+ key_expected = value_expected = false
+ state = :initial
+ else
+ raise_inspect "else case \" reached; %p not handled." % peek(1), tokens
+ end
+
+ else
+ raise_inspect 'Unknown state', tokens
+
+ end
+
+ match ||= matched
+ if $CODERAY_DEBUG and not kind
+ raise_inspect 'Error token %p in line %d' %
+ [[match, kind], line], tokens
+ end
+ raise_inspect 'Empty token', tokens unless match
+
+ tokens << [match, kind]
+
+ end
+
+ if [:string, :regexp].include? state
+ tokens << [:close, state]
+ end
+
+ tokens
+ end
+
+ protected
+
+ def reset_instance
+ super
+ @xml_scanner.reset if defined? @xml_scanner
+ end
+
+ def xml_scanner
+ @xml_scanner ||= CodeRay.scanner :xml, :tokens => @tokens, :keep_tokens => true, :keep_state => false
+ end
+
+ end
+
+end
+end
--- /dev/null
+module CodeRay
+module Scanners
+
+ class JavaScript < Scanner
+
+ include Streamable
+
+ register_for :java_script
+ file_extension 'js'
+
+ # The actual JavaScript keywords.
+ KEYWORDS = %w[
+ break case catch continue default delete do else
+ finally for function if in instanceof new
+ return switch throw try typeof var void while with
+ ]
+ PREDEFINED_CONSTANTS = %w[
+ false null true undefined
+ ]
+
+ MAGIC_VARIABLES = %w[ this arguments ] # arguments was introduced in JavaScript 1.4
+
+ KEYWORDS_EXPECTING_VALUE = WordList.new.add %w[
+ case delete in instanceof new return throw typeof with
+ ]
+
+ # Reserved for future use.
+ RESERVED_WORDS = %w[
+ abstract boolean byte char class debugger double enum export extends
+ final float goto implements import int interface long native package
+ private protected public short static super synchronized throws transient
+ volatile
+ ]
+
+ IDENT_KIND = WordList.new(:ident).
+ add(RESERVED_WORDS, :reserved).
+ add(PREDEFINED_CONSTANTS, :pre_constant).
+ add(MAGIC_VARIABLES, :local_variable).
+ add(KEYWORDS, :keyword)
+
+ ESCAPE = / [bfnrtv\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} /x
+ UNICODE_ESCAPE = / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} /x
+ REGEXP_ESCAPE = / [bBdDsSwW] /x
+ STRING_CONTENT_PATTERN = {
+ "'" => /[^\\']+/,
+ '"' => /[^\\"]+/,
+ '/' => /[^\\\/]+/,
+ }
+ KEY_CHECK_PATTERN = {
+ "'" => / (?> [^\\']* (?: \\. [^\\']* )* ) ' \s* : /mx,
+ '"' => / (?> [^\\"]* (?: \\. [^\\"]* )* ) " \s* : /mx,
+ }
+
+ def scan_tokens tokens, options
+
+ state = :initial
+ string_delimiter = nil
+ value_expected = true
+ key_expected = false
+ function_expected = false
+
+ until eos?
+
+ kind = nil
+ match = nil
+
+ case state
+
+ when :initial
+
+ if match = scan(/ \s+ | \\\n /x)
+ value_expected = true if !value_expected && match.index(?\n)
+ tokens << [match, :space]
+ next
+
+ elsif scan(%r! // [^\n\\]* (?: \\. [^\n\\]* )* | /\* (?: .*? \*/ | .* ) !mx)
+ value_expected = true
+ kind = :comment
+
+ elsif check(/\.?\d/)
+ key_expected = value_expected = false
+ if scan(/0[xX][0-9A-Fa-f]+/)
+ kind = :hex
+ elsif scan(/(?>0[0-7]+)(?![89.eEfF])/)
+ kind = :oct
+ elsif scan(/\d+[fF]|\d*\.\d+(?:[eE][+-]?\d+)?[fF]?|\d+[eE][+-]?\d+[fF]?/)
+ kind = :float
+ elsif scan(/\d+/)
+ kind = :integer
+ end
+
+ elsif value_expected && match = scan(/<([[:alpha:]]\w*) (?: [^\/>]*\/> | .*?<\/\1>)/xim)
+ # FIXME: scan over nested tags
+ xml_scanner.tokenize match
+ value_expected = false
+ next
+
+ elsif match = scan(/ [-+*=<>?:;,!&^|(\[{~%]+ | \.(?!\d) /x)
+ value_expected = true
+ last_operator = match[-1]
+ key_expected = (last_operator == ?{) || (last_operator == ?,)
+ function_expected = false
+ kind = :operator
+
+ elsif scan(/ [)\]}]+ /x)
+ function_expected = key_expected = value_expected = false
+ kind = :operator
+
+ elsif match = scan(/ [$a-zA-Z_][A-Za-z_0-9$]* /x)
+ kind = IDENT_KIND[match]
+ value_expected = (kind == :keyword) && KEYWORDS_EXPECTING_VALUE[match]
+ # TODO: labels
+ if kind == :ident
+ if match.index(?$) # $ allowed inside an identifier
+ kind = :predefined
+ elsif function_expected
+ kind = :function
+ elsif check(/\s*[=:]\s*function\b/)
+ kind = :function
+ elsif key_expected && check(/\s*:/)
+ kind = :key
+ end
+ end
+ function_expected = (kind == :keyword) && (match == 'function')
+ key_expected = false
+
+ elsif match = scan(/["']/)
+ if key_expected && check(KEY_CHECK_PATTERN[match])
+ state = :key
+ else
+ state = :string
+ end
+ tokens << [:open, state]
+ string_delimiter = match
+ kind = :delimiter
+
+ elsif value_expected && (match = scan(/\/(?=\S)/))
+ tokens << [:open, :regexp]
+ state = :regexp
+ string_delimiter = '/'
+ kind = :delimiter
+
+ elsif scan(/ \/ /x)
+ value_expected = true
+ key_expected = false
+ kind = :operator
+
+ else
+ getch
+ kind = :error
+
+ end
+
+ when :string, :regexp, :key
+ if scan(STRING_CONTENT_PATTERN[string_delimiter])
+ kind = :content
+ elsif match = scan(/["'\/]/)
+ tokens << [match, :delimiter]
+ if state == :regexp
+ modifiers = scan(/[gim]+/)
+ tokens << [modifiers, :modifier] if modifiers && !modifiers.empty?
+ end
+ tokens << [:close, state]
+ string_delimiter = nil
+ key_expected = value_expected = false
+ state = :initial
+ next
+ elsif state != :regexp && (match = scan(/ \\ (?: #{ESCAPE} | #{UNICODE_ESCAPE} ) /mox))
+ if string_delimiter == "'" && !(match == "\\\\" || match == "\\'")
+ kind = :content
+ else
+ kind = :char
+ end
+ elsif state == :regexp && scan(/ \\ (?: #{ESCAPE} | #{REGEXP_ESCAPE} | #{UNICODE_ESCAPE} ) /mox)
+ kind = :char
+ elsif scan(/\\./m)
+ kind = :content
+ elsif scan(/ \\ | $ /x)
+ tokens << [:close, state]
+ kind = :error
+ key_expected = value_expected = false
+ state = :initial
+ else
+ raise_inspect "else case \" reached; %p not handled." % peek(1), tokens
+ end
+
+ else
+ raise_inspect 'Unknown state', tokens
+
+ end
+
+ match ||= matched
+ if $CODERAY_DEBUG and not kind
+ raise_inspect 'Error token %p in line %d' %
+ [[match, kind], line], tokens
+ end
+ raise_inspect 'Empty token', tokens unless match
+
+ tokens << [match, kind]
+
+ end
+
+ if [:string, :regexp].include? state
+ tokens << [:close, state]
+ end
+
+ tokens
+ end
+
+ protected
+
+ def reset_instance
+ super
+ @xml_scanner.reset if defined? @xml_scanner
+ end
+
+ def xml_scanner
+ @xml_scanner ||= CodeRay.scanner :xml, :tokens => @tokens, :keep_tokens => true, :keep_state => false
+ end
+
+ end
+
+end
+end
--- /dev/null
+module CodeRay
+module Scanners
+
+ class JSON < Scanner
+
+ include Streamable
+
+ register_for :json
+ file_extension 'json'
+
+ KINDS_NOT_LOC = [
+ :float, :char, :content, :delimiter,
+ :error, :integer, :operator, :value,
+ ]
+
+ ESCAPE = / [bfnrt\\"\/] /x
+ UNICODE_ESCAPE = / u[a-fA-F0-9]{4} /x
+
+ def scan_tokens tokens, options
+
+ state = :initial
+ stack = []
+ key_expected = false
+
+ until eos?
+
+ kind = nil
+ match = nil
+
+ case state
+
+ when :initial
+ if match = scan(/ \s+ | \\\n /x)
+ tokens << [match, :space]
+ next
+ elsif match = scan(/ [:,\[{\]}] /x)
+ kind = :operator
+ case match
+ when '{' then stack << :object; key_expected = true
+ when '[' then stack << :array
+ when ':' then key_expected = false
+ when ',' then key_expected = true if stack.last == :object
+ when '}', ']' then stack.pop # no error recovery, but works for valid JSON
+ end
+ elsif match = scan(/ true | false | null /x)
+ kind = :value
+ elsif match = scan(/-?(?:0|[1-9]\d*)/)
+ kind = :integer
+ if scan(/\.\d+(?:[eE][-+]?\d+)?|[eE][-+]?\d+/)
+ match << matched
+ kind = :float
+ end
+ elsif match = scan(/"/)
+ state = key_expected ? :key : :string
+ tokens << [:open, state]
+ kind = :delimiter
+ else
+ getch
+ kind = :error
+ end
+
+ when :string, :key
+ if scan(/[^\\"]+/)
+ kind = :content
+ elsif scan(/"/)
+ tokens << ['"', :delimiter]
+ tokens << [:close, state]
+ state = :initial
+ next
+ elsif scan(/ \\ (?: #{ESCAPE} | #{UNICODE_ESCAPE} ) /mox)
+ kind = :char
+ elsif scan(/\\./m)
+ kind = :content
+ elsif scan(/ \\ | $ /x)
+ tokens << [:close, state]
+ kind = :error
+ state = :initial
+ else
+ raise_inspect "else case \" reached; %p not handled." % peek(1), tokens
+ end
+
+ else
+ raise_inspect 'Unknown state', tokens
+
+ end
+
+ match ||= matched
+ if $CODERAY_DEBUG and not kind
+ raise_inspect 'Error token %p in line %d' %
+ [[match, kind], line], tokens
+ end
+ raise_inspect 'Empty token', tokens unless match
+
+ tokens << [match, kind]
+
+ end
+
+ if [:string, :key].include? state
+ tokens << [:close, state]
+ end
+
+ tokens
+ end
+
+ end
+
+end
+end
--- /dev/null
+module CodeRay
+module Scanners
+
+ load :html
+ load :ruby
+
+ # Nitro XHTML Scanner
+ class NitroXHTML < Scanner
+
+ include Streamable
+ register_for :nitro_xhtml
+ file_extension :xhtml
+ title 'Nitro XHTML'
+
+ KINDS_NOT_LOC = HTML::KINDS_NOT_LOC
+
+ NITRO_RUBY_BLOCK = /
+ <\?r
+ (?>
+ [^\?]*
+ (?> \?(?!>) [^\?]* )*
+ )
+ (?: \?> )?
+ |
+ <ruby>
+ (?>
+ [^<]*
+ (?> <(?!\/ruby>) [^<]* )*
+ )
+ (?: <\/ruby> )?
+ |
+ <%
+ (?>
+ [^%]*
+ (?> %(?!>) [^%]* )*
+ )
+ (?: %> )?
+ /mx
+
+ NITRO_VALUE_BLOCK = /
+ \#
+ (?:
+ \{
+ [^{}]*
+ (?>
+ \{ [^}]* \}
+ (?> [^{}]* )
+ )*
+ \}?
+ | \| [^|]* \|?
+ | \( [^)]* \)?
+ | \[ [^\]]* \]?
+ | \\ [^\\]* \\?
+ )
+ /x
+
+ NITRO_ENTITY = /
+ % (?: \#\d+ | \w+ ) ;
+ /
+
+ START_OF_RUBY = /
+ (?=[<\#%])
+ < (?: \?r | % | ruby> )
+ | \# [{(|]
+ | % (?: \#\d+ | \w+ ) ;
+ /x
+
+ CLOSING_PAREN = Hash.new do |h, p|
+ h[p] = p
+ end.update( {
+ '(' => ')',
+ '[' => ']',
+ '{' => '}',
+ } )
+
+ private
+
+ def setup
+ @ruby_scanner = CodeRay.scanner :ruby, :tokens => @tokens, :keep_tokens => true
+ @html_scanner = CodeRay.scanner :html, :tokens => @tokens, :keep_tokens => true, :keep_state => true
+ end
+
+ def reset_instance
+ super
+ @html_scanner.reset
+ end
+
+ def scan_tokens tokens, options
+
+ until eos?
+
+ if (match = scan_until(/(?=#{START_OF_RUBY})/o) || scan_until(/\z/)) and not match.empty?
+ @html_scanner.tokenize match
+
+ elsif match = scan(/#{NITRO_VALUE_BLOCK}/o)
+ start_tag = match[0,2]
+ delimiter = CLOSING_PAREN[start_tag[1,1]]
+ end_tag = match[-1,1] == delimiter ? delimiter : ''
+ tokens << [:open, :inline]
+ tokens << [start_tag, :inline_delimiter]
+ code = match[start_tag.size .. -1 - end_tag.size]
+ @ruby_scanner.tokenize code
+ tokens << [end_tag, :inline_delimiter] unless end_tag.empty?
+ tokens << [:close, :inline]
+
+ elsif match = scan(/#{NITRO_RUBY_BLOCK}/o)
+ start_tag = '<?r'
+ end_tag = match[-2,2] == '?>' ? '?>' : ''
+ tokens << [:open, :inline]
+ tokens << [start_tag, :inline_delimiter]
+ code = match[start_tag.size .. -(end_tag.size)-1]
+ @ruby_scanner.tokenize code
+ tokens << [end_tag, :inline_delimiter] unless end_tag.empty?
+ tokens << [:close, :inline]
+
+ elsif entity = scan(/#{NITRO_ENTITY}/o)
+ tokens << [entity, :entity]
+
+ elsif scan(/%/)
+ tokens << [matched, :error]
+
+ else
+ raise_inspect 'else-case reached!', tokens
+
+ end
+
+ end
+
+ tokens
+
+ end
+
+ end
+
+end
+end
--- /dev/null
+module CodeRay
+module Scanners
+
+ load :html
+
+ # Original by Stefan Walk.
+ class PHP < Scanner
+
+ register_for :php
+ file_extension 'php'
+
+ KINDS_NOT_LOC = HTML::KINDS_NOT_LOC
+
+ def setup
+ @html_scanner = CodeRay.scanner :html, :tokens => @tokens, :keep_tokens => true, :keep_state => true
+ end
+
+ def reset_instance
+ super
+ @html_scanner.reset
+ end
+
+ module Words
+
+ # according to http://www.php.net/manual/en/reserved.keywords.php
+ KEYWORDS = %w[
+ abstract and array as break case catch class clone const continue declare default do else elseif
+ enddeclare endfor endforeach endif endswitch endwhile extends final for foreach function global
+ goto if implements interface instanceof namespace new or private protected public static switch
+ throw try use var while xor
+ cfunction old_function
+ ]
+
+ TYPES = %w[ int integer float double bool boolean string array object resource ]
+
+ LANGUAGE_CONSTRUCTS = %w[
+ die echo empty exit eval include include_once isset list
+ require require_once return print unset
+ ]
+
+ CLASSES = %w[ Directory stdClass __PHP_Incomplete_Class exception php_user_filter Closure ]
+
+ # according to http://php.net/quickref.php on 2009-04-21;
+ # all functions with _ excluded (module functions) and selected additional functions
+ BUILTIN_FUNCTIONS = %w[
+ abs acos acosh addcslashes addslashes aggregate array arsort ascii2ebcdic asin asinh asort assert atan atan2
+ atanh basename bcadd bccomp bcdiv bcmod bcmul bcpow bcpowmod bcscale bcsqrt bcsub bin2hex bindec
+ bindtextdomain bzclose bzcompress bzdecompress bzerrno bzerror bzerrstr bzflush bzopen bzread bzwrite
+ calculhmac ceil chdir checkdate checkdnsrr chgrp chmod chop chown chr chroot clearstatcache closedir closelog
+ compact constant copy cos cosh count crc32 crypt current date dcgettext dcngettext deaggregate decbin dechex
+ decoct define defined deg2rad delete dgettext die dirname diskfreespace dl dngettext doubleval each
+ ebcdic2ascii echo empty end ereg eregi escapeshellarg escapeshellcmd eval exec exit exp explode expm1 extract
+ fclose feof fflush fgetc fgetcsv fgets fgetss file fileatime filectime filegroup fileinode filemtime fileowner
+ fileperms filepro filesize filetype floatval flock floor flush fmod fnmatch fopen fpassthru fprintf fputcsv
+ fputs fread frenchtojd fscanf fseek fsockopen fstat ftell ftok ftruncate fwrite getallheaders getcwd getdate
+ getenv gethostbyaddr gethostbyname gethostbynamel getimagesize getlastmod getmxrr getmygid getmyinode getmypid
+ getmyuid getopt getprotobyname getprotobynumber getrandmax getrusage getservbyname getservbyport gettext
+ gettimeofday gettype glob gmdate gmmktime gmstrftime gregoriantojd gzclose gzcompress gzdecode gzdeflate
+ gzencode gzeof gzfile gzgetc gzgets gzgetss gzinflate gzopen gzpassthru gzputs gzread gzrewind gzseek gztell
+ gzuncompress gzwrite hash header hebrev hebrevc hexdec htmlentities htmlspecialchars hypot iconv idate
+ implode include intval ip2long iptcembed iptcparse isset
+ jddayofweek jdmonthname jdtofrench jdtogregorian jdtojewish jdtojulian jdtounix jewishtojd join jpeg2wbmp
+ juliantojd key krsort ksort lcfirst lchgrp lchown levenshtein link linkinfo list localeconv localtime log
+ log10 log1p long2ip lstat ltrim mail main max md5 metaphone mhash microtime min mkdir mktime msql natcasesort
+ natsort next ngettext nl2br nthmac octdec opendir openlog
+ ord overload pack passthru pathinfo pclose pfsockopen phpcredits phpinfo phpversion pi png2wbmp popen pos pow
+ prev print printf putenv quotemeta rad2deg rand range rawurldecode rawurlencode readdir readfile readgzfile
+ readline readlink realpath recode rename require reset rewind rewinddir rmdir round rsort rtrim scandir
+ serialize setcookie setlocale setrawcookie settype sha1 shuffle signeurlpaiement sin sinh sizeof sleep snmpget
+ snmpgetnext snmprealwalk snmpset snmpwalk snmpwalkoid sort soundex split spliti sprintf sqrt srand sscanf stat
+ strcasecmp strchr strcmp strcoll strcspn strftime stripcslashes stripos stripslashes stristr strlen
+ strnatcasecmp strnatcmp strncasecmp strncmp strpbrk strpos strptime strrchr strrev strripos strrpos strspn
+ strstr strtok strtolower strtotime strtoupper strtr strval substr symlink syslog system tan tanh tempnam
+ textdomain time tmpfile touch trim uasort ucfirst ucwords uksort umask uniqid unixtojd unlink unpack
+ unserialize unset urldecode urlencode usleep usort vfprintf virtual vprintf vsprintf wordwrap
+ array_change_key_case array_chunk array_combine array_count_values array_diff array_diff_assoc
+ array_diff_key array_diff_uassoc array_diff_ukey array_fill array_fill_keys array_filter array_flip
+ array_intersect array_intersect_assoc array_intersect_key array_intersect_uassoc array_intersect_ukey
+ array_key_exists array_keys array_map array_merge array_merge_recursive array_multisort array_pad
+ array_pop array_product array_push array_rand array_reduce array_reverse array_search array_shift
+ array_slice array_splice array_sum array_udiff array_udiff_assoc array_udiff_uassoc array_uintersect
+ array_uintersect_assoc array_uintersect_uassoc array_unique array_unshift array_values array_walk
+ array_walk_recursive
+ assert_options base_convert base64_decode base64_encode
+ chunk_split class_exists class_implements class_parents
+ count_chars debug_backtrace debug_print_backtrace debug_zval_dump
+ error_get_last error_log error_reporting extension_loaded
+ file_exists file_get_contents file_put_contents load_file
+ func_get_arg func_get_args func_num_args function_exists
+ get_browser get_called_class get_cfg_var get_class get_class_methods get_class_vars
+ get_current_user get_declared_classes get_declared_interfaces get_defined_constants
+ get_defined_functions get_defined_vars get_extension_funcs get_headers get_html_translation_table
+ get_include_path get_included_files get_loaded_extensions get_magic_quotes_gpc get_magic_quotes_runtime
+ get_meta_tags get_object_vars get_parent_class get_required_filesget_resource_type
+ gc_collect_cycles gc_disable gc_enable gc_enabled
+ halt_compiler headers_list headers_sent highlight_file highlight_string
+ html_entity_decode htmlspecialchars_decode
+ in_array include_once inclued_get_data
+ is_a is_array is_binary is_bool is_buffer is_callable is_dir is_double is_executable is_file is_finite
+ is_float is_infinite is_int is_integer is_link is_long is_nan is_null is_numeric is_object is_readable
+ is_real is_resource is_scalar is_soap_fault is_string is_subclass_of is_unicode is_uploaded_file
+ is_writable is_writeable
+ locale_get_default locale_set_default
+ number_format override_function parse_str parse_url
+ php_check_syntax php_ini_loaded_file php_ini_scanned_files php_logo_guid php_sapi_name
+ php_strip_whitespace php_uname
+ preg_filter preg_grep preg_last_error preg_match preg_match_all preg_quote preg_replace
+ preg_replace_callback preg_split print_r
+ require_once register_shutdown_function register_tick_function
+ set_error_handler set_exception_handler set_file_buffer set_include_path
+ set_magic_quotes_runtime set_time_limit shell_exec
+ str_getcsv str_ireplace str_pad str_repeat str_replace str_rot13 str_shuffle str_split str_word_count
+ strip_tags substr_compare substr_count substr_replace
+ time_nanosleep time_sleep_until
+ token_get_all token_name trigger_error
+ unregister_tick_function use_soap_error_handler user_error
+ utf8_decode utf8_encode var_dump var_export
+ version_compare
+ zend_logo_guid zend_thread_id zend_version
+ create_function call_user_func_array
+ posix_access posix_ctermid posix_get_last_error posix_getcwd posix_getegid
+ posix_geteuid posix_getgid posix_getgrgid posix_getgrnam posix_getgroups
+ posix_getlogin posix_getpgid posix_getpgrp posix_getpid posix_getppid
+ posix_getpwnam posix_getpwuid posix_getrlimit posix_getsid posix_getuid
+ posix_initgroups posix_isatty posix_kill posix_mkfifo posix_mknod
+ posix_setegid posix_seteuid posix_setgid posix_setpgid posix_setsid
+ posix_setuid posix_strerror posix_times posix_ttyname posix_uname
+ pcntl_alarm pcntl_exec pcntl_fork pcntl_getpriority pcntl_setpriority
+ pcntl_signal pcntl_signal_dispatch pcntl_sigprocmask pcntl_sigtimedwait
+ pcntl_sigwaitinfo pcntl_wait pcntl_waitpid pcntl_wexitstatus pcntl_wifexited
+ pcntl_wifsignaled pcntl_wifstopped pcntl_wstopsig pcntl_wtermsig
+ ]
+ # TODO: more built-in PHP functions?
+
+ EXCEPTIONS = %w[
+ E_ERROR E_WARNING E_PARSE E_NOTICE E_CORE_ERROR E_CORE_WARNING E_COMPILE_ERROR E_COMPILE_WARNING
+ E_USER_ERROR E_USER_WARNING E_USER_NOTICE E_DEPRECATED E_USER_DEPRECATED E_ALL E_STRICT
+ ]
+
+ CONSTANTS = %w[
+ null true false self parent
+ __LINE__ __DIR__ __FILE__ __LINE__
+ __CLASS__ __NAMESPACE__ __METHOD__ __FUNCTION__
+ PHP_VERSION PHP_MAJOR_VERSION PHP_MINOR_VERSION PHP_RELEASE_VERSION PHP_VERSION_ID PHP_EXTRA_VERSION PHP_ZTS
+ PHP_DEBUG PHP_MAXPATHLEN PHP_OS PHP_SAPI PHP_EOL PHP_INT_MAX PHP_INT_SIZE DEFAULT_INCLUDE_PATH
+ PEAR_INSTALL_DIR PEAR_EXTENSION_DIR PHP_EXTENSION_DIR PHP_PREFIX PHP_BINDIR PHP_LIBDIR PHP_DATADIR
+ PHP_SYSCONFDIR PHP_LOCALSTATEDIR PHP_CONFIG_FILE_PATH PHP_CONFIG_FILE_SCAN_DIR PHP_SHLIB_SUFFIX
+ PHP_OUTPUT_HANDLER_START PHP_OUTPUT_HANDLER_CONT PHP_OUTPUT_HANDLER_END
+ __COMPILER_HALT_OFFSET__
+ EXTR_OVERWRITE EXTR_SKIP EXTR_PREFIX_SAME EXTR_PREFIX_ALL EXTR_PREFIX_INVALID EXTR_PREFIX_IF_EXISTS
+ EXTR_IF_EXISTS SORT_ASC SORT_DESC SORT_REGULAR SORT_NUMERIC SORT_STRING CASE_LOWER CASE_UPPER COUNT_NORMAL
+ COUNT_RECURSIVE ASSERT_ACTIVE ASSERT_CALLBACK ASSERT_BAIL ASSERT_WARNING ASSERT_QUIET_EVAL CONNECTION_ABORTED
+ CONNECTION_NORMAL CONNECTION_TIMEOUT INI_USER INI_PERDIR INI_SYSTEM INI_ALL M_E M_LOG2E M_LOG10E M_LN2 M_LN10
+ M_PI M_PI_2 M_PI_4 M_1_PI M_2_PI M_2_SQRTPI M_SQRT2 M_SQRT1_2 CRYPT_SALT_LENGTH CRYPT_STD_DES CRYPT_EXT_DES
+ CRYPT_MD5 CRYPT_BLOWFISH DIRECTORY_SEPARATOR SEEK_SET SEEK_CUR SEEK_END LOCK_SH LOCK_EX LOCK_UN LOCK_NB
+ HTML_SPECIALCHARS HTML_ENTITIES ENT_COMPAT ENT_QUOTES ENT_NOQUOTES INFO_GENERAL INFO_CREDITS
+ INFO_CONFIGURATION INFO_MODULES INFO_ENVIRONMENT INFO_VARIABLES INFO_LICENSE INFO_ALL CREDITS_GROUP
+ CREDITS_GENERAL CREDITS_SAPI CREDITS_MODULES CREDITS_DOCS CREDITS_FULLPAGE CREDITS_QA CREDITS_ALL STR_PAD_LEFT
+ STR_PAD_RIGHT STR_PAD_BOTH PATHINFO_DIRNAME PATHINFO_BASENAME PATHINFO_EXTENSION PATH_SEPARATOR CHAR_MAX
+ LC_CTYPE LC_NUMERIC LC_TIME LC_COLLATE LC_MONETARY LC_ALL LC_MESSAGES ABDAY_1 ABDAY_2 ABDAY_3 ABDAY_4 ABDAY_5
+ ABDAY_6 ABDAY_7 DAY_1 DAY_2 DAY_3 DAY_4 DAY_5 DAY_6 DAY_7 ABMON_1 ABMON_2 ABMON_3 ABMON_4 ABMON_5 ABMON_6
+ ABMON_7 ABMON_8 ABMON_9 ABMON_10 ABMON_11 ABMON_12 MON_1 MON_2 MON_3 MON_4 MON_5 MON_6 MON_7 MON_8 MON_9
+ MON_10 MON_11 MON_12 AM_STR PM_STR D_T_FMT D_FMT T_FMT T_FMT_AMPM ERA ERA_YEAR ERA_D_T_FMT ERA_D_FMT ERA_T_FMT
+ ALT_DIGITS INT_CURR_SYMBOL CURRENCY_SYMBOL CRNCYSTR MON_DECIMAL_POINT MON_THOUSANDS_SEP MON_GROUPING
+ POSITIVE_SIGN NEGATIVE_SIGN INT_FRAC_DIGITS FRAC_DIGITS P_CS_PRECEDES P_SEP_BY_SPACE N_CS_PRECEDES
+ N_SEP_BY_SPACE P_SIGN_POSN N_SIGN_POSN DECIMAL_POINT RADIXCHAR THOUSANDS_SEP THOUSEP GROUPING YESEXPR NOEXPR
+ YESSTR NOSTR CODESET LOG_EMERG LOG_ALERT LOG_CRIT LOG_ERR LOG_WARNING LOG_NOTICE LOG_INFO LOG_DEBUG LOG_KERN
+ LOG_USER LOG_MAIL LOG_DAEMON LOG_AUTH LOG_SYSLOG LOG_LPR LOG_NEWS LOG_UUCP LOG_CRON LOG_AUTHPRIV LOG_LOCAL0
+ LOG_LOCAL1 LOG_LOCAL2 LOG_LOCAL3 LOG_LOCAL4 LOG_LOCAL5 LOG_LOCAL6 LOG_LOCAL7 LOG_PID LOG_CONS LOG_ODELAY
+ LOG_NDELAY LOG_NOWAIT LOG_PERROR
+ ]
+
+ PREDEFINED = %w[
+ $GLOBALS $_SERVER $_GET $_POST $_FILES $_REQUEST $_SESSION $_ENV
+ $_COOKIE $php_errormsg $HTTP_RAW_POST_DATA $http_response_header
+ $argc $argv
+ ]
+
+ IDENT_KIND = CaseIgnoringWordList.new(:ident).
+ add(KEYWORDS, :reserved).
+ add(TYPES, :pre_type).
+ add(LANGUAGE_CONSTRUCTS, :reserved).
+ add(BUILTIN_FUNCTIONS, :predefined).
+ add(CLASSES, :pre_constant).
+ add(EXCEPTIONS, :exception).
+ add(CONSTANTS, :pre_constant)
+
+ VARIABLE_KIND = WordList.new(:local_variable).
+ add(PREDEFINED, :predefined)
+ end
+
+ module RE
+
+ PHP_START = /
+ <script\s+[^>]*?language\s*=\s*"php"[^>]*?> |
+ <script\s+[^>]*?language\s*=\s*'php'[^>]*?> |
+ <\?php\d? |
+ <\?(?!xml)
+ /xi
+
+ PHP_END = %r!
+ </script> |
+ \?>
+ !xi
+
+ HTML_INDICATOR = /<!DOCTYPE html|<(?:html|body|div|p)[> ]/i
+
+ IDENTIFIER = /[a-z_\x7f-\xFF][a-z0-9_\x7f-\xFF]*/i
+ VARIABLE = /\$#{IDENTIFIER}/
+
+ OPERATOR = /
+ \.(?!\d)=? | # dot that is not decimal point, string concatenation
+ && | \|\| | # logic
+ :: | -> | => | # scope, member, dictionary
+ \\(?!\n) | # namespace
+ \+\+ | -- | # increment, decrement
+ [,;?:()\[\]{}] | # simple delimiters
+ [-+*\/%&|^]=? | # ordinary math, binary logic, assignment shortcuts
+ [~$] | # whatever
+ =& | # reference assignment
+ [=!]=?=? | <> | # comparison and assignment
+ <<=? | >>=? | [<>]=? # comparison and shift
+ /x
+
+ end
+
+ def scan_tokens tokens, options
+ if string.respond_to?(:encoding)
+ unless string.encoding == Encoding::ASCII_8BIT
+ self.string = string.encode Encoding::ASCII_8BIT,
+ :invalid => :replace, :undef => :replace, :replace => '?'
+ end
+ end
+
+ if check(RE::PHP_START) || # starts with <?
+ (match?(/\s*<\S/) && exist?(RE::PHP_START)) || # starts with tag and contains <?
+ exist?(RE::HTML_INDICATOR) ||
+ check(/.{1,100}#{RE::PHP_START}/om) # PHP start after max 100 chars
+ # is HTML with embedded PHP, so start with HTML
+ states = [:initial]
+ else
+ # is just PHP, so start with PHP surrounded by HTML
+ states = [:initial, :php]
+ end
+
+ label_expected = true
+ case_expected = false
+
+ heredoc_delimiter = nil
+ delimiter = nil
+ modifier = nil
+
+ until eos?
+
+ match = nil
+ kind = nil
+
+ case states.last
+
+ when :initial # HTML
+ if scan RE::PHP_START
+ kind = :inline_delimiter
+ label_expected = true
+ states << :php
+ else
+ match = scan_until(/(?=#{RE::PHP_START})/o) || scan_until(/\z/)
+ @html_scanner.tokenize match unless match.empty?
+ next
+ end
+
+ when :php
+ if match = scan(/\s+/)
+ tokens << [match, :space]
+ next
+
+ elsif scan(%r! (?m: \/\* (?: .*? \*\/ | .* ) ) | (?://|\#) .*? (?=#{RE::PHP_END}|$) !xo)
+ kind = :comment
+
+ elsif match = scan(RE::IDENTIFIER)
+ kind = Words::IDENT_KIND[match]
+ if kind == :ident && label_expected && check(/:(?!:)/)
+ kind = :label
+ label_expected = true
+ else
+ label_expected = false
+ if kind == :ident && match =~ /^[A-Z]/
+ kind = :constant
+ elsif kind == :reserved
+ case match
+ when 'class'
+ states << :class_expected
+ when 'function'
+ states << :function_expected
+ when 'case', 'default'
+ case_expected = true
+ end
+ elsif match == 'b' && check(/['"]/) # binary string literal
+ modifier = match
+ next
+ end
+ end
+
+ elsif scan(/(?:\d+\.\d*|\d*\.\d+)(?:e[-+]?\d+)?|\d+e[-+]?\d+/i)
+ label_expected = false
+ kind = :float
+
+ elsif scan(/0x[0-9a-fA-F]+/)
+ label_expected = false
+ kind = :hex
+
+ elsif scan(/\d+/)
+ label_expected = false
+ kind = :integer
+
+ elsif scan(/'/)
+ tokens << [:open, :string]
+ if modifier
+ tokens << [modifier, :modifier]
+ modifier = nil
+ end
+ kind = :delimiter
+ states.push :sqstring
+
+ elsif match = scan(/["`]/)
+ tokens << [:open, :string]
+ if modifier
+ tokens << [modifier, :modifier]
+ modifier = nil
+ end
+ delimiter = match
+ kind = :delimiter
+ states.push :dqstring
+
+ elsif match = scan(RE::VARIABLE)
+ label_expected = false
+ kind = Words::VARIABLE_KIND[match]
+
+ elsif scan(/\{/)
+ kind = :operator
+ label_expected = true
+ states.push :php
+
+ elsif scan(/\}/)
+ if states.size == 1
+ kind = :error
+ else
+ states.pop
+ if states.last.is_a?(::Array)
+ delimiter = states.last[1]
+ states[-1] = states.last[0]
+ tokens << [matched, :delimiter]
+ tokens << [:close, :inline]
+ next
+ else
+ kind = :operator
+ label_expected = true
+ end
+ end
+
+ elsif scan(/@/)
+ label_expected = false
+ kind = :exception
+
+ elsif scan RE::PHP_END
+ kind = :inline_delimiter
+ states = [:initial]
+
+ elsif match = scan(/<<<(?:(#{RE::IDENTIFIER})|"(#{RE::IDENTIFIER})"|'(#{RE::IDENTIFIER})')/o)
+ tokens << [:open, :string]
+ warn 'heredoc in heredoc?' if heredoc_delimiter
+ heredoc_delimiter = Regexp.escape(self[1] || self[2] || self[3])
+ kind = :delimiter
+ states.push self[3] ? :sqstring : :dqstring
+ heredoc_delimiter = /#{heredoc_delimiter}(?=;?$)/
+
+ elsif match = scan(/#{RE::OPERATOR}/o)
+ label_expected = match == ';'
+ if case_expected
+ label_expected = true if match == ':'
+ case_expected = false
+ end
+ kind = :operator
+
+ else
+ getch
+ kind = :error
+
+ end
+
+ when :sqstring
+ if scan(heredoc_delimiter ? /[^\\\n]+/ : /[^'\\]+/)
+ kind = :content
+ elsif !heredoc_delimiter && scan(/'/)
+ tokens << [matched, :delimiter]
+ tokens << [:close, :string]
+ delimiter = nil
+ label_expected = false
+ states.pop
+ next
+ elsif heredoc_delimiter && match = scan(/\n/)
+ kind = :content
+ if scan heredoc_delimiter
+ tokens << ["\n", :content]
+ tokens << [matched, :delimiter]
+ tokens << [:close, :string]
+ heredoc_delimiter = nil
+ label_expected = false
+ states.pop
+ next
+ end
+ elsif scan(heredoc_delimiter ? /\\\\/ : /\\[\\'\n]/)
+ kind = :char
+ elsif scan(/\\./m)
+ kind = :content
+ elsif scan(/\\/)
+ kind = :error
+ end
+
+ when :dqstring
+ if scan(heredoc_delimiter ? /[^${\\\n]+/ : (delimiter == '"' ? /[^"${\\]+/ : /[^`${\\]+/))
+ kind = :content
+ elsif !heredoc_delimiter && scan(delimiter == '"' ? /"/ : /`/)
+ tokens << [matched, :delimiter]
+ tokens << [:close, :string]
+ delimiter = nil
+ label_expected = false
+ states.pop
+ next
+ elsif heredoc_delimiter && match = scan(/\n/)
+ kind = :content
+ if scan heredoc_delimiter
+ tokens << ["\n", :content]
+ tokens << [matched, :delimiter]
+ tokens << [:close, :string]
+ heredoc_delimiter = nil
+ label_expected = false
+ states.pop
+ next
+ end
+ elsif scan(/\\(?:x[0-9A-Fa-f]{1,2}|[0-7]{1,3})/)
+ kind = :char
+ elsif scan(heredoc_delimiter ? /\\[nrtvf\\$]/ : (delimiter == '"' ? /\\[nrtvf\\$"]/ : /\\[nrtvf\\$`]/))
+ kind = :char
+ elsif scan(/\\./m)
+ kind = :content
+ elsif scan(/\\/)
+ kind = :error
+ elsif match = scan(/#{RE::VARIABLE}/o)
+ kind = :local_variable
+ if check(/\[#{RE::IDENTIFIER}\]/o)
+ tokens << [:open, :inline]
+ tokens << [match, :local_variable]
+ tokens << [scan(/\[/), :operator]
+ tokens << [scan(/#{RE::IDENTIFIER}/o), :ident]
+ tokens << [scan(/\]/), :operator]
+ tokens << [:close, :inline]
+ next
+ elsif check(/\[/)
+ match << scan(/\[['"]?#{RE::IDENTIFIER}?['"]?\]?/o)
+ kind = :error
+ elsif check(/->#{RE::IDENTIFIER}/o)
+ tokens << [:open, :inline]
+ tokens << [match, :local_variable]
+ tokens << [scan(/->/), :operator]
+ tokens << [scan(/#{RE::IDENTIFIER}/o), :ident]
+ tokens << [:close, :inline]
+ next
+ elsif check(/->/)
+ match << scan(/->/)
+ kind = :error
+ end
+ elsif match = scan(/\{/)
+ if check(/\$/)
+ kind = :delimiter
+ states[-1] = [states.last, delimiter]
+ delimiter = nil
+ states.push :php
+ tokens << [:open, :inline]
+ else
+ kind = :string
+ end
+ elsif scan(/\$\{#{RE::IDENTIFIER}\}/o)
+ kind = :local_variable
+ elsif scan(/\$/)
+ kind = :content
+ end
+
+ when :class_expected
+ if scan(/\s+/)
+ kind = :space
+ elsif match = scan(/#{RE::IDENTIFIER}/o)
+ kind = :class
+ states.pop
+ else
+ states.pop
+ next
+ end
+
+ when :function_expected
+ if scan(/\s+/)
+ kind = :space
+ elsif scan(/&/)
+ kind = :operator
+ elsif match = scan(/#{RE::IDENTIFIER}/o)
+ kind = :function
+ states.pop
+ else
+ states.pop
+ next
+ end
+
+ else
+ raise_inspect 'Unknown state!', tokens, states
+ end
+
+ match ||= matched
+ if $CODERAY_DEBUG and not kind
+ raise_inspect 'Error token %p in line %d' %
+ [[match, kind], line], tokens, states
+ end
+ raise_inspect 'Empty token', tokens, states unless match
+
+ tokens << [match, kind]
+
+ end
+
+ tokens
+ end
+
+ end
+
+end
+end
--- /dev/null
+module CodeRay
+module Scanners
+
+ class Plaintext < Scanner
+
+ register_for :plaintext, :plain
+ title 'Plain text'
+
+ include Streamable
+
+ KINDS_NOT_LOC = [:plain]
+
+ def scan_tokens tokens, options
+ text = (scan_until(/\z/) || '')
+ tokens << [text, :plain]
+ end
+
+ end
+
+end
+end
--- /dev/null
+module CodeRay
+module Scanners
+
+ # Bases on pygments' PythonLexer, see
+ # http://dev.pocoo.org/projects/pygments/browser/pygments/lexers/agile.py.
+ class Python < Scanner
+
+ include Streamable
+
+ register_for :python
+ file_extension 'py'
+
+ KEYWORDS = [
+ 'and', 'as', 'assert', 'break', 'class', 'continue', 'def',
+ 'del', 'elif', 'else', 'except', 'finally', 'for',
+ 'from', 'global', 'if', 'import', 'in', 'is', 'lambda', 'not',
+ 'or', 'pass', 'raise', 'return', 'try', 'while', 'with', 'yield',
+ 'nonlocal', # new in Python 3
+ ]
+
+ OLD_KEYWORDS = [
+ 'exec', 'print', # gone in Python 3
+ ]
+
+ PREDEFINED_METHODS_AND_TYPES = %w[
+ __import__ abs all any apply basestring bin bool buffer
+ bytearray bytes callable chr classmethod cmp coerce compile
+ complex delattr dict dir divmod enumerate eval execfile exit
+ file filter float frozenset getattr globals hasattr hash hex id
+ input int intern isinstance issubclass iter len list locals
+ long map max min next object oct open ord pow property range
+ raw_input reduce reload repr reversed round set setattr slice
+ sorted staticmethod str sum super tuple type unichr unicode
+ vars xrange zip
+ ]
+
+ PREDEFINED_EXCEPTIONS = %w[
+ ArithmeticError AssertionError AttributeError
+ BaseException DeprecationWarning EOFError EnvironmentError
+ Exception FloatingPointError FutureWarning GeneratorExit IOError
+ ImportError ImportWarning IndentationError IndexError KeyError
+ KeyboardInterrupt LookupError MemoryError NameError
+ NotImplemented NotImplementedError OSError OverflowError
+ OverflowWarning PendingDeprecationWarning ReferenceError
+ RuntimeError RuntimeWarning StandardError StopIteration
+ SyntaxError SyntaxWarning SystemError SystemExit TabError
+ TypeError UnboundLocalError UnicodeDecodeError
+ UnicodeEncodeError UnicodeError UnicodeTranslateError
+ UnicodeWarning UserWarning ValueError Warning ZeroDivisionError
+ ]
+
+ PREDEFINED_VARIABLES_AND_CONSTANTS = [
+ 'False', 'True', 'None', # "keywords" since Python 3
+ 'self', 'Ellipsis', 'NotImplemented',
+ ]
+
+ IDENT_KIND = WordList.new(:ident).
+ add(KEYWORDS, :keyword).
+ add(OLD_KEYWORDS, :old_keyword).
+ add(PREDEFINED_METHODS_AND_TYPES, :predefined).
+ add(PREDEFINED_VARIABLES_AND_CONSTANTS, :pre_constant).
+ add(PREDEFINED_EXCEPTIONS, :exception)
+
+ NAME = / [^\W\d] \w* /x
+ ESCAPE = / [abfnrtv\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} /x
+ UNICODE_ESCAPE = / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} | N\{[-\w ]+\} /x
+
+ OPERATOR = /
+ \.\.\. | # ellipsis
+ \.(?!\d) | # dot but not decimal point
+ [,;:()\[\]{}] | # simple delimiters
+ \/\/=? | \*\*=? | # special math
+ [-+*\/%&|^]=? | # ordinary math and binary logic
+ [~`] | # binary complement and inspection
+ <<=? | >>=? | [<>=]=? | != # comparison and assignment
+ /x
+
+ STRING_DELIMITER_REGEXP = Hash.new do |h, delimiter|
+ h[delimiter] = Regexp.union delimiter
+ end
+
+ STRING_CONTENT_REGEXP = Hash.new do |h, delimiter|
+ h[delimiter] = / [^\\\n]+? (?= \\ | $ | #{Regexp.escape(delimiter)} ) /x
+ end
+
+ DEF_NEW_STATE = WordList.new(:initial).
+ add(%w(def), :def_expected).
+ add(%w(import from), :include_expected).
+ add(%w(class), :class_expected)
+
+ DESCRIPTOR = /
+ #{NAME}
+ (?: \. #{NAME} )*
+ | \*
+ /x
+
+ def scan_tokens tokens, options
+
+ state = :initial
+ string_delimiter = nil
+ string_raw = false
+ import_clause = class_name_follows = last_token_dot = false
+ unicode = string.respond_to?(:encoding) && string.encoding.name == 'UTF-8'
+ from_import_state = []
+
+ until eos?
+
+ kind = nil
+ match = nil
+
+ if state == :string
+ if scan(STRING_DELIMITER_REGEXP[string_delimiter])
+ tokens << [matched, :delimiter]
+ tokens << [:close, :string]
+ state = :initial
+ next
+ elsif string_delimiter.size == 3 && scan(/\n/)
+ kind = :content
+ elsif scan(STRING_CONTENT_REGEXP[string_delimiter])
+ kind = :content
+ elsif !string_raw && scan(/ \\ #{ESCAPE} /ox)
+ kind = :char
+ elsif scan(/ \\ #{UNICODE_ESCAPE} /ox)
+ kind = :char
+ elsif scan(/ \\ . /x)
+ kind = :content
+ elsif scan(/ \\ | $ /x)
+ tokens << [:close, :string]
+ kind = :error
+ state = :initial
+ else
+ raise_inspect "else case \" reached; %p not handled." % peek(1), tokens, state
+ end
+
+ elsif match = scan(/ [ \t]+ | \\\n /x)
+ tokens << [match, :space]
+ next
+
+ elsif match = scan(/\n/)
+ tokens << [match, :space]
+ state = :initial if state == :include_expected
+ next
+
+ elsif match = scan(/ \# [^\n]* /mx)
+ tokens << [match, :comment]
+ next
+
+ elsif state == :initial
+
+ if scan(/#{OPERATOR}/o)
+ kind = :operator
+
+ elsif match = scan(/(u?r?|b)?("""|"|'''|')/i)
+ tokens << [:open, :string]
+ string_delimiter = self[2]
+ string_raw = false
+ modifiers = self[1]
+ unless modifiers.empty?
+ string_raw = !!modifiers.index(?r)
+ tokens << [modifiers, :modifier]
+ match = string_delimiter
+ end
+ state = :string
+ kind = :delimiter
+
+ # TODO: backticks
+
+ elsif match = scan(unicode ? /#{NAME}/uo : /#{NAME}/o)
+ kind = IDENT_KIND[match]
+ # TODO: keyword arguments
+ kind = :ident if last_token_dot
+ if kind == :old_keyword
+ kind = check(/\(/) ? :ident : :keyword
+ elsif kind == :predefined && check(/ *=/)
+ kind = :ident
+ elsif kind == :keyword
+ state = DEF_NEW_STATE[match]
+ from_import_state << match.to_sym if state == :include_expected
+ end
+
+ elsif scan(/@[a-zA-Z0-9_.]+[lL]?/)
+ kind = :decorator
+
+ elsif scan(/0[xX][0-9A-Fa-f]+[lL]?/)
+ kind = :hex
+
+ elsif scan(/0[bB][01]+[lL]?/)
+ kind = :bin
+
+ elsif match = scan(/(?:\d*\.\d+|\d+\.\d*)(?:[eE][+-]?\d+)?|\d+[eE][+-]?\d+/)
+ kind = :float
+ if scan(/[jJ]/)
+ match << matched
+ kind = :imaginary
+ end
+
+ elsif scan(/0[oO][0-7]+|0[0-7]+(?![89.eE])[lL]?/)
+ kind = :oct
+
+ elsif match = scan(/\d+([lL])?/)
+ kind = :integer
+ if self[1] == nil && scan(/[jJ]/)
+ match << matched
+ kind = :imaginary
+ end
+
+ else
+ getch
+ kind = :error
+
+ end
+
+ elsif state == :def_expected
+ state = :initial
+ if match = scan(unicode ? /#{NAME}/uo : /#{NAME}/o)
+ kind = :method
+ else
+ next
+ end
+
+ elsif state == :class_expected
+ state = :initial
+ if match = scan(unicode ? /#{NAME}/uo : /#{NAME}/o)
+ kind = :class
+ else
+ next
+ end
+
+ elsif state == :include_expected
+ if match = scan(unicode ? /#{DESCRIPTOR}/uo : /#{DESCRIPTOR}/o)
+ kind = :include
+ if match == 'as'
+ kind = :keyword
+ from_import_state << :as
+ elsif from_import_state.first == :from && match == 'import'
+ kind = :keyword
+ from_import_state << :import
+ elsif from_import_state.last == :as
+ # kind = match[0,1][unicode ? /[[:upper:]]/u : /[[:upper:]]/] ? :class : :method
+ kind = :ident
+ from_import_state.pop
+ elsif IDENT_KIND[match] == :keyword
+ unscan
+ match = nil
+ state = :initial
+ next
+ end
+ elsif match = scan(/,/)
+ from_import_state.pop if from_import_state.last == :as
+ kind = :operator
+ else
+ from_import_state = []
+ state = :initial
+ next
+ end
+
+ else
+ raise_inspect 'Unknown state', tokens, state
+
+ end
+
+ match ||= matched
+ if $CODERAY_DEBUG and not kind
+ raise_inspect 'Error token %p in line %d' %
+ [[match, kind], line], tokens, state
+ end
+ raise_inspect 'Empty token', tokens, state unless match
+
+ last_token_dot = match == '.'
+
+ tokens << [match, kind]
+
+ end
+
+ if state == :string
+ tokens << [:close, :string]
+ end
+
+ tokens
+ end
+
+ end
+
+end
+end
--- /dev/null
+module CodeRay
+module Scanners
+
+ load :html
+ load :ruby
+
+ # RHTML Scanner
+ class RHTML < Scanner
+
+ include Streamable
+ register_for :rhtml
+ title 'HTML ERB Template'
+
+ KINDS_NOT_LOC = HTML::KINDS_NOT_LOC
+
+ ERB_RUBY_BLOCK = /
+ <%(?!%)[=-]?
+ (?>
+ [^\-%]* # normal*
+ (?> # special
+ (?: %(?!>) | -(?!%>) )
+ [^\-%]* # normal*
+ )*
+ )
+ (?: -?%> )?
+ /x
+
+ START_OF_ERB = /
+ <%(?!%)
+ /x
+
+ private
+
+ def setup
+ @ruby_scanner = CodeRay.scanner :ruby, :tokens => @tokens, :keep_tokens => true
+ @html_scanner = CodeRay.scanner :html, :tokens => @tokens, :keep_tokens => true, :keep_state => true
+ end
+
+ def reset_instance
+ super
+ @html_scanner.reset
+ end
+
+ def scan_tokens tokens, options
+
+ until eos?
+
+ if (match = scan_until(/(?=#{START_OF_ERB})/o) || scan_until(/\z/)) and not match.empty?
+ @html_scanner.tokenize match
+
+ elsif match = scan(/#{ERB_RUBY_BLOCK}/o)
+ start_tag = match[/\A<%[-=#]?/]
+ end_tag = match[/-?%?>?\z/]
+ tokens << [:open, :inline]
+ tokens << [start_tag, :inline_delimiter]
+ code = match[start_tag.size .. -1 - end_tag.size]
+ if start_tag == '<%#'
+ tokens << [code, :comment]
+ else
+ @ruby_scanner.tokenize code
+ end
+ tokens << [end_tag, :inline_delimiter] unless end_tag.empty?
+ tokens << [:close, :inline]
+
+ else
+ raise_inspect 'else-case reached!', tokens
+ end
+
+ end
+
+ tokens
+
+ end
+
+ end
+
+end
+end
--- /dev/null
+# encoding: utf-8
+module CodeRay
+module Scanners
+
+ # This scanner is really complex, since Ruby _is_ a complex language!
+ #
+ # It tries to highlight 100% of all common code,
+ # and 90% of strange codes.
+ #
+ # It is optimized for HTML highlighting, and is not very useful for
+ # parsing or pretty printing.
+ #
+ # For now, I think it's better than the scanners in VIM or Syntax, or
+ # any highlighter I was able to find, except Caleb's RubyLexer.
+ #
+ # I hope it's also better than the rdoc/irb lexer.
+ class Ruby < Scanner
+
+ include Streamable
+
+ register_for :ruby
+ file_extension 'rb'
+
+ helper :patterns
+
+ if not defined? EncodingError
+ EncodingError = Class.new Exception
+ end
+
+ private
+ def scan_tokens tokens, options
+ if string.respond_to?(:encoding)
+ unless string.encoding == Encoding::UTF_8
+ self.string = string.encode Encoding::UTF_8,
+ :invalid => :replace, :undef => :replace, :replace => '?'
+ end
+ unicode = false
+ else
+ unicode = exist?(/[^\x00-\x7f]/)
+ end
+
+ last_token_dot = false
+ value_expected = true
+ heredocs = nil
+ last_state = nil
+ state = :initial
+ depth = nil
+ inline_block_stack = []
+
+
+ patterns = Patterns # avoid constant lookup
+
+ until eos?
+ match = nil
+ kind = nil
+
+ if state.instance_of? patterns::StringState
+# {{{
+ match = scan_until(state.pattern) || scan_until(/\z/)
+ tokens << [match, :content] unless match.empty?
+ break if eos?
+
+ if state.heredoc and self[1] # end of heredoc
+ match = getch.to_s
+ match << scan_until(/$/) unless eos?
+ tokens << [match, :delimiter]
+ tokens << [:close, state.type]
+ state = state.next_state
+ next
+ end
+
+ case match = getch
+
+ when state.delim
+ if state.paren
+ state.paren_depth -= 1
+ if state.paren_depth > 0
+ tokens << [match, :nesting_delimiter]
+ next
+ end
+ end
+ tokens << [match, :delimiter]
+ if state.type == :regexp and not eos?
+ modifiers = scan(/#{patterns::REGEXP_MODIFIERS}/ox)
+ tokens << [modifiers, :modifier] unless modifiers.empty?
+ end
+ tokens << [:close, state.type]
+ value_expected = false
+ state = state.next_state
+
+ when '\\'
+ if state.interpreted
+ if esc = scan(/ #{patterns::ESCAPE} /ox)
+ tokens << [match + esc, :char]
+ else
+ tokens << [match, :error]
+ end
+ else
+ case m = getch
+ when state.delim, '\\'
+ tokens << [match + m, :char]
+ when nil
+ tokens << [match, :error]
+ else
+ tokens << [match + m, :content]
+ end
+ end
+
+ when '#'
+ case peek(1)
+ when '{'
+ inline_block_stack << [state, depth, heredocs]
+ value_expected = true
+ state = :initial
+ depth = 1
+ tokens << [:open, :inline]
+ tokens << [match + getch, :inline_delimiter]
+ when '$', '@'
+ tokens << [match, :escape]
+ last_state = state # scan one token as normal code, then return here
+ state = :initial
+ else
+ raise_inspect 'else-case # reached; #%p not handled' % peek(1), tokens
+ end
+
+ when state.paren
+ state.paren_depth += 1
+ tokens << [match, :nesting_delimiter]
+
+ when /#{patterns::REGEXP_SYMBOLS}/ox
+ tokens << [match, :function]
+
+ else
+ raise_inspect 'else-case " reached; %p not handled, state = %p' % [match, state], tokens
+
+ end
+ next
+# }}}
+ else
+# {{{
+ if match = scan(/[ \t\f]+/)
+ kind = :space
+ match << scan(/\s*/) unless eos? || heredocs
+ value_expected = true if match.index(?\n)
+ tokens << [match, kind]
+ next
+
+ elsif match = scan(/\\?\n/)
+ kind = :space
+ if match == "\n"
+ value_expected = true
+ state = :initial if state == :undef_comma_expected
+ end
+ if heredocs
+ unscan # heredoc scanning needs \n at start
+ state = heredocs.shift
+ tokens << [:open, state.type]
+ heredocs = nil if heredocs.empty?
+ next
+ else
+ match << scan(/\s*/) unless eos?
+ end
+ tokens << [match, kind]
+ next
+
+ elsif bol? && match = scan(/\#!.*/)
+ tokens << [match, :doctype]
+ next
+
+ elsif match = scan(/\#.*/) or
+ ( bol? and match = scan(/#{patterns::RUBYDOC_OR_DATA}/o) )
+ kind = :comment
+ tokens << [match, kind]
+ next
+
+ elsif state == :initial
+
+ # IDENTS #
+ if match = scan(unicode ? /#{patterns::METHOD_NAME}/uo :
+ /#{patterns::METHOD_NAME}/o)
+ if last_token_dot
+ kind = if match[/^[A-Z]/] and not match?(/\(/) then :constant else :ident end
+ else
+ if value_expected != :expect_colon && scan(/:(?= )/)
+ tokens << [match, :key]
+ match = ':'
+ kind = :operator
+ else
+ kind = patterns::IDENT_KIND[match]
+ if kind == :ident
+ if match[/\A[A-Z]/] and not match[/[!?]$/] and not match?(/\(/)
+ kind = :constant
+ end
+ elsif kind == :reserved
+ state = patterns::DEF_NEW_STATE[match]
+ value_expected = :set if patterns::KEYWORDS_EXPECTING_VALUE[match]
+ end
+ end
+ end
+ value_expected = :set if check(/#{patterns::VALUE_FOLLOWS}/o)
+
+ elsif last_token_dot and match = scan(/#{patterns::METHOD_NAME_OPERATOR}|\(/o)
+ kind = :ident
+ value_expected = :set if check(unicode ? /#{patterns::VALUE_FOLLOWS}/uo :
+ /#{patterns::VALUE_FOLLOWS}/o)
+
+ # OPERATORS #
+ elsif not last_token_dot and match = scan(/ \.\.\.? | (?:\.|::)() | [,\(\)\[\]\{\}] | ==?=? /x)
+ if match !~ / [.\)\]\}] /x or match =~ /\.\.\.?/
+ value_expected = :set
+ end
+ last_token_dot = :set if self[1]
+ kind = :operator
+ unless inline_block_stack.empty?
+ case match
+ when '{'
+ depth += 1
+ when '}'
+ depth -= 1
+ if depth == 0 # closing brace of inline block reached
+ state, depth, heredocs = inline_block_stack.pop
+ heredocs = nil if heredocs && heredocs.empty?
+ tokens << [match, :inline_delimiter]
+ kind = :inline
+ match = :close
+ end
+ end
+ end
+
+ elsif match = scan(/ ['"] /mx)
+ tokens << [:open, :string]
+ kind = :delimiter
+ state = patterns::StringState.new :string, match == '"', match # important for streaming
+
+ elsif match = scan(unicode ? /#{patterns::INSTANCE_VARIABLE}/uo :
+ /#{patterns::INSTANCE_VARIABLE}/o)
+ kind = :instance_variable
+
+ elsif value_expected and match = scan(/\//)
+ tokens << [:open, :regexp]
+ kind = :delimiter
+ interpreted = true
+ state = patterns::StringState.new :regexp, interpreted, match
+
+ # elsif match = scan(/[-+]?#{patterns::NUMERIC}/o)
+ elsif match = value_expected ? scan(/[-+]?#{patterns::NUMERIC}/o) : scan(/#{patterns::NUMERIC}/o)
+ kind = self[1] ? :float : :integer
+
+ elsif match = scan(unicode ? /#{patterns::SYMBOL}/uo :
+ /#{patterns::SYMBOL}/o)
+ case delim = match[1]
+ when ?', ?"
+ tokens << [:open, :symbol]
+ tokens << [':', :symbol]
+ match = delim.chr
+ kind = :delimiter
+ state = patterns::StringState.new :symbol, delim == ?", match
+ else
+ kind = :symbol
+ end
+
+ elsif match = scan(/ -[>=]? | [+!~^]=? | [*|&]{1,2}=? | >>? /x)
+ value_expected = :set
+ kind = :operator
+
+ elsif value_expected and match = scan(unicode ? /#{patterns::HEREDOC_OPEN}/uo :
+ /#{patterns::HEREDOC_OPEN}/o)
+ indented = self[1] == '-'
+ quote = self[3]
+ delim = self[quote ? 4 : 2]
+ kind = patterns::QUOTE_TO_TYPE[quote]
+ tokens << [:open, kind]
+ tokens << [match, :delimiter]
+ match = :close
+ heredoc = patterns::StringState.new kind, quote != '\'', delim, (indented ? :indented : :linestart )
+ heredocs ||= [] # create heredocs if empty
+ heredocs << heredoc
+
+ elsif value_expected and match = scan(/#{patterns::FANCY_START_CORRECT}/o)
+ kind, interpreted = *patterns::FancyStringType.fetch(self[1]) do
+ raise_inspect 'Unknown fancy string: %%%p' % k, tokens
+ end
+ tokens << [:open, kind]
+ state = patterns::StringState.new kind, interpreted, self[2]
+ kind = :delimiter
+
+ elsif value_expected and match = scan(unicode ? /#{patterns::CHARACTER}/uo :
+ /#{patterns::CHARACTER}/o)
+ kind = :integer
+
+ elsif match = scan(/ [\/%]=? | <(?:<|=>?)? | [?:;] /x)
+ value_expected = :set
+ kind = :operator
+
+ elsif match = scan(/`/)
+ if last_token_dot
+ kind = :operator
+ else
+ tokens << [:open, :shell]
+ kind = :delimiter
+ state = patterns::StringState.new :shell, true, match
+ end
+
+ elsif match = scan(unicode ? /#{patterns::GLOBAL_VARIABLE}/uo :
+ /#{patterns::GLOBAL_VARIABLE}/o)
+ kind = :global_variable
+
+ elsif match = scan(unicode ? /#{patterns::CLASS_VARIABLE}/uo :
+ /#{patterns::CLASS_VARIABLE}/o)
+ kind = :class_variable
+
+ else
+ if !unicode && !string.respond_to?(:encoding)
+ # check for unicode
+ debug, $DEBUG = $DEBUG, false
+ begin
+ if check(/./mu).size > 1
+ # seems like we should try again with unicode
+ unicode = true
+ end
+ rescue
+ # bad unicode char; use getch
+ ensure
+ $DEBUG = debug
+ end
+ next if unicode
+ end
+ kind = :error
+ match = scan(unicode ? /./mu : /./m)
+
+ end
+
+ elsif state == :def_expected
+ state = :initial
+ if scan(/self\./)
+ tokens << ['self', :pre_constant]
+ tokens << ['.', :operator]
+ end
+ if match = scan(unicode ? /(?>#{patterns::METHOD_NAME_EX})(?!\.|::)/uo :
+ /(?>#{patterns::METHOD_NAME_EX})(?!\.|::)/o)
+ kind = :method
+ else
+ next
+ end
+
+ elsif state == :module_expected
+ if match = scan(/<</)
+ kind = :operator
+ else
+ state = :initial
+ if match = scan(unicode ? /(?:#{patterns::IDENT}::)*#{patterns::IDENT}/uo :
+ /(?:#{patterns::IDENT}::)*#{patterns::IDENT}/o)
+ kind = :class
+ else
+ next
+ end
+ end
+
+ elsif state == :undef_expected
+ state = :undef_comma_expected
+ if match = scan(unicode ? /#{patterns::METHOD_NAME_EX}/uo :
+ /#{patterns::METHOD_NAME_EX}/o)
+ kind = :method
+ elsif match = scan(unicode ? /#{patterns::SYMBOL}/uo :
+ /#{patterns::SYMBOL}/o)
+ case delim = match[1]
+ when ?', ?"
+ tokens << [:open, :symbol]
+ tokens << [':', :symbol]
+ match = delim.chr
+ kind = :delimiter
+ state = patterns::StringState.new :symbol, delim == ?", match
+ state.next_state = :undef_comma_expected
+ else
+ kind = :symbol
+ end
+ else
+ state = :initial
+ next
+ end
+
+ elsif state == :alias_expected
+ match = scan(unicode ? /(#{patterns::METHOD_NAME_OR_SYMBOL})([ \t]+)(#{patterns::METHOD_NAME_OR_SYMBOL})/uo :
+ /(#{patterns::METHOD_NAME_OR_SYMBOL})([ \t]+)(#{patterns::METHOD_NAME_OR_SYMBOL})/o)
+
+ if match
+ tokens << [self[1], (self[1][0] == ?: ? :symbol : :method)]
+ tokens << [self[2], :space]
+ tokens << [self[3], (self[3][0] == ?: ? :symbol : :method)]
+ end
+ state = :initial
+ next
+
+ elsif state == :undef_comma_expected
+ if match = scan(/,/)
+ kind = :operator
+ state = :undef_expected
+ else
+ state = :initial
+ next
+ end
+
+ end
+# }}}
+
+ unless kind == :error
+ if value_expected = value_expected == :set
+ value_expected = :expect_colon if match == '?' || match == 'when'
+ end
+ last_token_dot = last_token_dot == :set
+ end
+
+ if $CODERAY_DEBUG and not kind
+ raise_inspect 'Error token %p in line %d' %
+ [[match, kind], line], tokens, state
+ end
+ raise_inspect 'Empty token', tokens unless match
+
+ tokens << [match, kind]
+
+ if last_state
+ state = last_state
+ last_state = nil
+ end
+ end
+ end
+
+ inline_block_stack << [state] if state.is_a? patterns::StringState
+ until inline_block_stack.empty?
+ this_block = inline_block_stack.pop
+ tokens << [:close, :inline] if this_block.size > 1
+ state = this_block.first
+ tokens << [:close, state.type]
+ end
+
+ tokens
+ end
+
+ end
+
+end
+end
+
+# vim:fdm=marker
--- /dev/null
+# encoding: utf-8
+module CodeRay
+module Scanners
+
+ module Ruby::Patterns # :nodoc:
+
+ RESERVED_WORDS = %w[
+ and def end in or unless begin
+ defined? ensure module redo super until
+ BEGIN break do next rescue then
+ when END case else for retry
+ while alias class elsif if not return
+ undef yield
+ ]
+
+ DEF_KEYWORDS = %w[ def ]
+ UNDEF_KEYWORDS = %w[ undef ]
+ ALIAS_KEYWORDS = %w[ alias ]
+ MODULE_KEYWORDS = %w[ class module ]
+ DEF_NEW_STATE = WordList.new(:initial).
+ add(DEF_KEYWORDS, :def_expected).
+ add(UNDEF_KEYWORDS, :undef_expected).
+ add(ALIAS_KEYWORDS, :alias_expected).
+ add(MODULE_KEYWORDS, :module_expected)
+
+ PREDEFINED_CONSTANTS = %w[
+ nil true false self
+ DATA ARGV ARGF
+ __FILE__ __LINE__ __ENCODING__
+ ]
+
+ IDENT_KIND = WordList.new(:ident).
+ add(RESERVED_WORDS, :reserved).
+ add(PREDEFINED_CONSTANTS, :pre_constant)
+
+ if /\w/u === '∑'
+ # MRI 1.8.6, 1.8.7
+ IDENT = /[^\W\d]\w*/
+ else
+ if //.respond_to? :encoding
+ # MRI 1.9.1, 1.9.2
+ IDENT = Regexp.new '[\p{L}\p{M}\p{Pc}\p{Sm}&&[^\x00-\x40\x5b-\x5e\x60\x7b-\x7f]][\p{L}\p{M}\p{N}\p{Pc}\p{Sm}&&[^\x00-\x2f\x3a-\x40\x5b-\x5e\x60\x7b-\x7f]]*'
+ else
+ # JRuby, Rubinius
+ IDENT = /[^\x00-\x40\x5b-\x5e\x60\x7b-\x7f][^\x00-\x2f\x3a-\x40\x5b-\x5e\x60\x7b-\x7f]*/
+ end
+ end
+
+ METHOD_NAME = / #{IDENT} [?!]? /ox
+ METHOD_NAME_OPERATOR = /
+ \*\*? # multiplication and power
+ | [-+~]@? # plus, minus, tilde with and without at sign
+ | [\/%&|^`] # division, modulo or format strings, and, or, xor, system
+ | \[\]=? # array getter and setter
+ | << | >> # append or shift left, shift right
+ | <=?>? | >=? # comparison, rocket operator
+ | ===? | =~ # simple equality, case equality, match
+ | ![~=@]? # negation with and without at sign, not-equal and not-match
+ /ox
+ METHOD_NAME_EX = / #{IDENT} (?:[?!]|=(?!>))? | #{METHOD_NAME_OPERATOR} /ox
+ INSTANCE_VARIABLE = / @ #{IDENT} /ox
+ CLASS_VARIABLE = / @@ #{IDENT} /ox
+ OBJECT_VARIABLE = / @@? #{IDENT} /ox
+ GLOBAL_VARIABLE = / \$ (?: #{IDENT} | [1-9]\d* | 0\w* | [~&+`'=\/,;_.<>!@$?*":\\] | -[a-zA-Z_0-9] ) /ox
+ PREFIX_VARIABLE = / #{GLOBAL_VARIABLE} | #{OBJECT_VARIABLE} /ox
+ VARIABLE = / @?@? #{IDENT} | #{GLOBAL_VARIABLE} /ox
+
+ QUOTE_TO_TYPE = {
+ '`' => :shell,
+ '/'=> :regexp,
+ }
+ QUOTE_TO_TYPE.default = :string
+
+ REGEXP_MODIFIERS = /[mixounse]*/
+ REGEXP_SYMBOLS = /[|?*+(){}\[\].^$]/
+
+ DECIMAL = /\d+(?:_\d+)*/
+ OCTAL = /0_?[0-7]+(?:_[0-7]+)*/
+ HEXADECIMAL = /0x[0-9A-Fa-f]+(?:_[0-9A-Fa-f]+)*/
+ BINARY = /0b[01]+(?:_[01]+)*/
+
+ EXPONENT = / [eE] [+-]? #{DECIMAL} /ox
+ FLOAT_SUFFIX = / #{EXPONENT} | \. #{DECIMAL} #{EXPONENT}? /ox
+ FLOAT_OR_INT = / #{DECIMAL} (?: #{FLOAT_SUFFIX} () )? /ox
+ NUMERIC = / (?: (?=0) (?: #{OCTAL} | #{HEXADECIMAL} | #{BINARY} ) | #{FLOAT_OR_INT} ) /ox
+
+ SYMBOL = /
+ :
+ (?:
+ #{METHOD_NAME_EX}
+ | #{PREFIX_VARIABLE}
+ | ['"]
+ )
+ /ox
+ METHOD_NAME_OR_SYMBOL = / #{METHOD_NAME_EX} | #{SYMBOL} /ox
+
+ SIMPLE_ESCAPE = /
+ [abefnrstv]
+ | [0-7]{1,3}
+ | x[0-9A-Fa-f]{1,2}
+ | .?
+ /mx
+
+ CONTROL_META_ESCAPE = /
+ (?: M-|C-|c )
+ (?: \\ (?: M-|C-|c ) )*
+ (?: [^\\] | \\ #{SIMPLE_ESCAPE} )?
+ /mox
+
+ ESCAPE = /
+ #{CONTROL_META_ESCAPE} | #{SIMPLE_ESCAPE}
+ /mox
+
+ CHARACTER = /
+ \?
+ (?:
+ [^\s\\]
+ | \\ #{ESCAPE}
+ )
+ /mox
+
+ # NOTE: This is not completely correct, but
+ # nobody needs heredoc delimiters ending with \n.
+ # Also, delimiters starting with numbers are allowed.
+ # but they are more often than not a false positive.
+ HEREDOC_OPEN = /
+ << (-)? # $1 = float
+ (?:
+ ( #{IDENT} ) # $2 = delim
+ |
+ ( ["'`\/] ) # $3 = quote, type
+ ( [^\n]*? ) \3 # $4 = delim
+ )
+ /mx
+
+ RUBYDOC = /
+ =begin (?!\S)
+ .*?
+ (?: \Z | ^=end (?!\S) [^\n]* )
+ /mx
+
+ DATA = /
+ __END__$
+ .*?
+ (?: \Z | (?=^\#CODE) )
+ /mx
+
+ # Checks for a valid value to follow. This enables
+ # value_expected in method calls without parentheses.
+ VALUE_FOLLOWS = /
+ (?>[ \t\f\v]+)
+ (?:
+ [%\/][^\s=]
+ | <<-?\S
+ | [-+] \d
+ | #{CHARACTER}
+ )
+ /x
+ KEYWORDS_EXPECTING_VALUE = WordList.new.add(%w[
+ and end in or unless begin
+ defined? ensure redo super until
+ break do next rescue then
+ when case else for retry
+ while elsif if not return
+ yield
+ ])
+
+ RUBYDOC_OR_DATA = / #{RUBYDOC} | #{DATA} /xo
+
+ RDOC_DATA_START = / ^=begin (?!\S) | ^__END__$ /x
+
+ FANCY_START_CORRECT = / % ( [qQwWxsr] | (?![a-zA-Z0-9]) ) ([^a-zA-Z0-9]) /mx
+
+ FancyStringType = {
+ 'q' => [:string, false],
+ 'Q' => [:string, true],
+ 'r' => [:regexp, true],
+ 's' => [:symbol, false],
+ 'x' => [:shell, true]
+ }
+ FancyStringType['w'] = FancyStringType['q']
+ FancyStringType['W'] = FancyStringType[''] = FancyStringType['Q']
+
+ class StringState < Struct.new :type, :interpreted, :delim, :heredoc,
+ :paren, :paren_depth, :pattern, :next_state
+
+ CLOSING_PAREN = Hash[ *%w[
+ ( )
+ [ ]
+ < >
+ { }
+ ] ]
+
+ CLOSING_PAREN.each { |k,v| k.freeze; v.freeze } # debug, if I try to change it with <<
+ OPENING_PAREN = CLOSING_PAREN.invert
+
+ STRING_PATTERN = Hash.new do |h, k|
+ delim, interpreted = *k
+ delim_pattern = Regexp.escape(delim.dup) # dup: workaround for old Ruby
+ if closing_paren = CLOSING_PAREN[delim]
+ delim_pattern = delim_pattern[0..-1] if defined? JRUBY_VERSION # JRuby fix
+ delim_pattern << Regexp.escape(closing_paren)
+ end
+ delim_pattern << '\\\\' unless delim == '\\'
+
+ special_escapes =
+ case interpreted
+ when :regexp_symbols
+ '| ' + REGEXP_SYMBOLS.source
+ when :words
+ '| \s'
+ end
+
+ h[k] =
+ if interpreted and not delim == '#'
+ / (?= [#{delim_pattern}] | \# [{$@] #{special_escapes} ) /mx
+ else
+ / (?= [#{delim_pattern}] #{special_escapes} ) /mx
+ end
+ end
+
+ HEREDOC_PATTERN = Hash.new do |h, k|
+ delim, interpreted, indented = *k
+ delim_pattern = Regexp.escape(delim.dup) # dup: workaround for old Ruby
+ delim_pattern = / \n #{ '(?>[\ \t]*)' if indented } #{ Regexp.new delim_pattern } $ /x
+ h[k] =
+ if interpreted
+ / (?= #{delim_pattern}() | \\ | \# [{$@] ) /mx # $1 set == end of heredoc
+ else
+ / (?= #{delim_pattern}() | \\ ) /mx
+ end
+ end
+
+ def initialize kind, interpreted, delim, heredoc = false
+ if heredoc
+ pattern = HEREDOC_PATTERN[ [delim, interpreted, heredoc == :indented] ]
+ delim = nil
+ else
+ pattern = STRING_PATTERN[ [delim, interpreted] ]
+ if paren = CLOSING_PAREN[delim]
+ delim, paren = paren, delim
+ paren_depth = 1
+ end
+ end
+ super kind, interpreted, delim, heredoc, paren, paren_depth, pattern, :initial
+ end
+ end unless defined? StringState
+
+ end
+
+end
+end
--- /dev/null
+module CodeRay
+ module Scanners
+
+ # Scheme scanner for CodeRay (by closure).
+ # Thanks to murphy for putting CodeRay into public.
+ class Scheme < Scanner
+
+ # TODO: function defs
+ # TODO: built-in functions
+
+ register_for :scheme
+ file_extension 'scm'
+
+ CORE_FORMS = %w[
+ lambda let let* letrec syntax-case define-syntax let-syntax
+ letrec-syntax begin define quote if or and cond case do delay
+ quasiquote set! cons force call-with-current-continuation call/cc
+ ]
+
+ IDENT_KIND = CaseIgnoringWordList.new(:ident).
+ add(CORE_FORMS, :reserved)
+
+ #IDENTIFIER_INITIAL = /[a-z!@\$%&\*\/\:<=>\?~_\^]/i
+ #IDENTIFIER_SUBSEQUENT = /#{IDENTIFIER_INITIAL}|\d|\.|\+|-/
+ #IDENTIFIER = /#{IDENTIFIER_INITIAL}#{IDENTIFIER_SUBSEQUENT}*|\+|-|\.{3}/
+ IDENTIFIER = /[a-zA-Z!@$%&*\/:<=>?~_^][\w!@$%&*\/:<=>?~^.+\-]*|[+-]|\.\.\./
+ DIGIT = /\d/
+ DIGIT10 = DIGIT
+ DIGIT16 = /[0-9a-f]/i
+ DIGIT8 = /[0-7]/
+ DIGIT2 = /[01]/
+ RADIX16 = /\#x/i
+ RADIX8 = /\#o/i
+ RADIX2 = /\#b/i
+ RADIX10 = /\#d/i
+ EXACTNESS = /#i|#e/i
+ SIGN = /[\+-]?/
+ EXP_MARK = /[esfdl]/i
+ EXP = /#{EXP_MARK}#{SIGN}#{DIGIT}+/
+ SUFFIX = /#{EXP}?/
+ PREFIX10 = /#{RADIX10}?#{EXACTNESS}?|#{EXACTNESS}?#{RADIX10}?/
+ PREFIX16 = /#{RADIX16}#{EXACTNESS}?|#{EXACTNESS}?#{RADIX16}/
+ PREFIX8 = /#{RADIX8}#{EXACTNESS}?|#{EXACTNESS}?#{RADIX8}/
+ PREFIX2 = /#{RADIX2}#{EXACTNESS}?|#{EXACTNESS}?#{RADIX2}/
+ UINT10 = /#{DIGIT10}+#*/
+ UINT16 = /#{DIGIT16}+#*/
+ UINT8 = /#{DIGIT8}+#*/
+ UINT2 = /#{DIGIT2}+#*/
+ DECIMAL = /#{DIGIT10}+#+\.#*#{SUFFIX}|#{DIGIT10}+\.#{DIGIT10}*#*#{SUFFIX}|\.#{DIGIT10}+#*#{SUFFIX}|#{UINT10}#{EXP}/
+ UREAL10 = /#{UINT10}\/#{UINT10}|#{DECIMAL}|#{UINT10}/
+ UREAL16 = /#{UINT16}\/#{UINT16}|#{UINT16}/
+ UREAL8 = /#{UINT8}\/#{UINT8}|#{UINT8}/
+ UREAL2 = /#{UINT2}\/#{UINT2}|#{UINT2}/
+ REAL10 = /#{SIGN}#{UREAL10}/
+ REAL16 = /#{SIGN}#{UREAL16}/
+ REAL8 = /#{SIGN}#{UREAL8}/
+ REAL2 = /#{SIGN}#{UREAL2}/
+ IMAG10 = /i|#{UREAL10}i/
+ IMAG16 = /i|#{UREAL16}i/
+ IMAG8 = /i|#{UREAL8}i/
+ IMAG2 = /i|#{UREAL2}i/
+ COMPLEX10 = /#{REAL10}@#{REAL10}|#{REAL10}\+#{IMAG10}|#{REAL10}-#{IMAG10}|\+#{IMAG10}|-#{IMAG10}|#{REAL10}/
+ COMPLEX16 = /#{REAL16}@#{REAL16}|#{REAL16}\+#{IMAG16}|#{REAL16}-#{IMAG16}|\+#{IMAG16}|-#{IMAG16}|#{REAL16}/
+ COMPLEX8 = /#{REAL8}@#{REAL8}|#{REAL8}\+#{IMAG8}|#{REAL8}-#{IMAG8}|\+#{IMAG8}|-#{IMAG8}|#{REAL8}/
+ COMPLEX2 = /#{REAL2}@#{REAL2}|#{REAL2}\+#{IMAG2}|#{REAL2}-#{IMAG2}|\+#{IMAG2}|-#{IMAG2}|#{REAL2}/
+ NUM10 = /#{PREFIX10}?#{COMPLEX10}/
+ NUM16 = /#{PREFIX16}#{COMPLEX16}/
+ NUM8 = /#{PREFIX8}#{COMPLEX8}/
+ NUM2 = /#{PREFIX2}#{COMPLEX2}/
+ NUM = /#{NUM10}|#{NUM16}|#{NUM8}|#{NUM2}/
+
+ private
+ def scan_tokens tokens,options
+
+ state = :initial
+ ident_kind = IDENT_KIND
+
+ until eos?
+ kind = match = nil
+
+ case state
+ when :initial
+ if scan(/ \s+ | \\\n /x)
+ kind = :space
+ elsif scan(/['\(\[\)\]]|#\(/)
+ kind = :operator_fat
+ elsif scan(/;.*/)
+ kind = :comment
+ elsif scan(/#\\(?:newline|space|.?)/)
+ kind = :char
+ elsif scan(/#[ft]/)
+ kind = :pre_constant
+ elsif scan(/#{IDENTIFIER}/o)
+ kind = ident_kind[matched]
+ elsif scan(/\./)
+ kind = :operator
+ elsif scan(/"/)
+ tokens << [:open, :string]
+ state = :string
+ tokens << ['"', :delimiter]
+ next
+ elsif scan(/#{NUM}/o) and not matched.empty?
+ kind = :integer
+ elsif getch
+ kind = :error
+ end
+
+ when :string
+ if scan(/[^"\\]+/) or scan(/\\.?/)
+ kind = :content
+ elsif scan(/"/)
+ tokens << ['"', :delimiter]
+ tokens << [:close, :string]
+ state = :initial
+ next
+ else
+ raise_inspect "else case \" reached; %p not handled." % peek(1),
+ tokens, state
+ end
+
+ else
+ raise "else case reached"
+ end
+
+ match ||= matched
+ if $CODERAY_DEBUG and not kind
+ raise_inspect 'Error token %p in line %d' %
+ [[match, kind], line], tokens
+ end
+ raise_inspect 'Empty token', tokens, state unless match
+
+ tokens << [match, kind]
+
+ end # until eos
+
+ if state == :string
+ tokens << [:close, :string]
+ end
+
+ tokens
+
+ end #scan_tokens
+ end #class
+ end #module scanners
+end #module coderay
\ No newline at end of file
--- /dev/null
+module CodeRay module Scanners
+
+ # by Josh Goebel
+ class SQL < Scanner
+
+ register_for :sql
+
+ RESERVED_WORDS = %w(
+ create database table index trigger drop primary key set select
+ insert update delete replace into
+ on from values before and or if exists case when
+ then else as group order by avg where
+ join inner outer union engine not
+ like end using collate show columns begin
+ )
+
+ PREDEFINED_TYPES = %w(
+ char varchar enum binary text tinytext mediumtext
+ longtext blob tinyblob mediumblob longblob timestamp
+ date time datetime year double decimal float int
+ integer tinyint mediumint bigint smallint unsigned bit
+ bool boolean hex bin oct
+ )
+
+ PREDEFINED_FUNCTIONS = %w( sum cast abs pi count min max avg )
+
+ DIRECTIVES = %w( auto_increment unique default charset )
+
+ PREDEFINED_CONSTANTS = %w( null true false )
+
+ IDENT_KIND = CaseIgnoringWordList.new(:ident).
+ add(RESERVED_WORDS, :reserved).
+ add(PREDEFINED_TYPES, :pre_type).
+ add(PREDEFINED_CONSTANTS, :pre_constant).
+ add(PREDEFINED_FUNCTIONS, :predefined).
+ add(DIRECTIVES, :directive)
+
+ ESCAPE = / [rbfntv\n\\\/'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} | . /mx
+ UNICODE_ESCAPE = / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} /x
+
+ STRING_PREFIXES = /[xnb]|_\w+/i
+
+ def scan_tokens tokens, options
+
+ state = :initial
+ string_type = nil
+ string_content = ''
+
+ until eos?
+
+ kind = nil
+ match = nil
+
+ if state == :initial
+
+ if scan(/ \s+ | \\\n /x)
+ kind = :space
+
+ elsif scan(/(?:--\s?|#).*/)
+ kind = :comment
+
+ elsif scan(%r! /\* (?: .*? \*/ | .* ) !mx)
+ kind = :comment
+
+ elsif scan(/ [-+*\/=<>;,!&^|()\[\]{}~%] | \.(?!\d) /x)
+ kind = :operator
+
+ elsif scan(/(#{STRING_PREFIXES})?([`"'])/o)
+ prefix = self[1]
+ string_type = self[2]
+ tokens << [:open, :string]
+ tokens << [prefix, :modifier] if prefix
+ match = string_type
+ state = :string
+ kind = :delimiter
+
+ elsif match = scan(/ @? [A-Za-z_][A-Za-z_0-9]* /x)
+ kind = match[0] == ?@ ? :variable : IDENT_KIND[match.downcase]
+
+ elsif scan(/0[xX][0-9A-Fa-f]+/)
+ kind = :hex
+
+ elsif scan(/0[0-7]+(?![89.eEfF])/)
+ kind = :oct
+
+ elsif scan(/(?>\d+)(?![.eEfF])/)
+ kind = :integer
+
+ elsif scan(/\d[fF]|\d*\.\d+(?:[eE][+-]?\d+)?|\d+[eE][+-]?\d+/)
+ kind = :float
+
+ else
+ getch
+ kind = :error
+
+ end
+
+ elsif state == :string
+ if match = scan(/[^\\"'`]+/)
+ string_content << match
+ next
+ elsif match = scan(/["'`]/)
+ if string_type == match
+ if peek(1) == string_type # doubling means escape
+ string_content << string_type << getch
+ next
+ end
+ unless string_content.empty?
+ tokens << [string_content, :content]
+ string_content = ''
+ end
+ tokens << [matched, :delimiter]
+ tokens << [:close, :string]
+ state = :initial
+ string_type = nil
+ next
+ else
+ string_content << match
+ end
+ next
+ elsif scan(/ \\ (?: #{ESCAPE} | #{UNICODE_ESCAPE} ) /mox)
+ unless string_content.empty?
+ tokens << [string_content, :content]
+ string_content = ''
+ end
+ kind = :char
+ elsif match = scan(/ \\ . /mox)
+ string_content << match
+ next
+ elsif scan(/ \\ | $ /x)
+ unless string_content.empty?
+ tokens << [string_content, :content]
+ string_content = ''
+ end
+ kind = :error
+ state = :initial
+ else
+ raise "else case \" reached; %p not handled." % peek(1), tokens
+ end
+
+ else
+ raise 'else-case reached', tokens
+
+ end
+
+ match ||= matched
+ unless kind
+ raise_inspect 'Error token %p in line %d' %
+ [[match, kind], line], tokens, state
+ end
+ raise_inspect 'Empty token', tokens unless match
+
+ tokens << [match, kind]
+
+ end
+ tokens
+
+ end
+
+ end
+
+end end
\ No newline at end of file
--- /dev/null
+module CodeRay
+module Scanners
+
+ load :html
+
+ # XML Scanner
+ #
+ # Currently this is the same scanner as Scanners::HTML.
+ class XML < HTML
+
+ register_for :xml
+ file_extension 'xml'
+
+ end
+
+end
+end
--- /dev/null
+module CodeRay
+module Scanners
+
+ # YAML Scanner
+ #
+ # Based on the YAML scanner from Syntax by Jamis Buck.
+ class YAML < Scanner
+
+ register_for :yaml
+ file_extension 'yml'
+
+ KINDS_NOT_LOC = :all
+
+ def scan_tokens tokens, options
+
+ value_expected = nil
+ state = :initial
+ key_indent = indent = 0
+
+ until eos?
+
+ kind = nil
+ match = nil
+ key_indent = nil if bol?
+
+ if match = scan(/ +[\t ]*/)
+ kind = :space
+
+ elsif match = scan(/\n+/)
+ kind = :space
+ state = :initial if match.index(?\n)
+
+ elsif match = scan(/#.*/)
+ kind = :comment
+
+ elsif bol? and case
+ when match = scan(/---|\.\.\./)
+ tokens << [:open, :head]
+ tokens << [match, :head]
+ tokens << [:close, :head]
+ next
+ when match = scan(/%.*/)
+ tokens << [match, :doctype]
+ next
+ end
+
+ elsif state == :value and case
+ when !check(/(?:"[^"]*")(?=: |:$)/) && scan(/"/)
+ tokens << [:open, :string]
+ tokens << [matched, :delimiter]
+ tokens << [matched, :content] if scan(/ [^"\\]* (?: \\. [^"\\]* )* /mx)
+ tokens << [matched, :delimiter] if scan(/"/)
+ tokens << [:close, :string]
+ next
+ when match = scan(/[|>][-+]?/)
+ tokens << [:open, :string]
+ tokens << [match, :delimiter]
+ string_indent = key_indent || column(pos - match.size - 1)
+ tokens << [matched, :content] if scan(/(?:\n+ {#{string_indent + 1}}.*)+/)
+ tokens << [:close, :string]
+ next
+ when match = scan(/(?![!"*&]).+?(?=$|\s+#)/)
+ tokens << [match, :string]
+ string_indent = key_indent || column(pos - match.size - 1)
+ tokens << [matched, :string] if scan(/(?:\n+ {#{string_indent + 1}}.*)+/)
+ next
+ end
+
+ elsif case
+ when match = scan(/[-:](?= |$)/)
+ state = :value if state == :colon && (match == ':' || match == '-')
+ state = :value if state == :initial && match == '-'
+ kind = :operator
+ when match = scan(/[,{}\[\]]/)
+ kind = :operator
+ when state == :initial && match = scan(/[\w.() ]*\S(?=: |:$)/)
+ kind = :key
+ key_indent = column(pos - match.size - 1)
+ # tokens << [key_indent.inspect, :debug]
+ state = :colon
+ when match = scan(/(?:"[^"\n]*"|'[^'\n]*')(?=: |:$)/)
+ tokens << [:open, :key]
+ tokens << [match[0,1], :delimiter]
+ tokens << [match[1..-2], :content]
+ tokens << [match[-1,1], :delimiter]
+ tokens << [:close, :key]
+ key_indent = column(pos - match.size - 1)
+ # tokens << [key_indent.inspect, :debug]
+ state = :colon
+ next
+ when scan(/(![\w\/]+)(:([\w:]+))?/)
+ tokens << [self[1], :type]
+ if self[2]
+ tokens << [':', :operator]
+ tokens << [self[3], :class]
+ end
+ next
+ when scan(/&\S+/)
+ kind = :variable
+ when scan(/\*\w+/)
+ kind = :global_variable
+ when scan(/<</)
+ kind = :class_variable
+ when scan(/\d\d:\d\d:\d\d/)
+ kind = :oct
+ when scan(/\d\d\d\d-\d\d-\d\d\s\d\d:\d\d:\d\d(\.\d+)? [-+]\d\d:\d\d/)
+ kind = :oct
+ when scan(/:\w+/)
+ kind = :symbol
+ when scan(/[^:\s]+(:(?! |$)[^:\s]*)* .*/)
+ kind = :error
+ when scan(/[^:\s]+(:(?! |$)[^:\s]*)*/)
+ kind = :error
+ end
+
+ else
+ getch
+ kind = :error
+
+ end
+
+ match ||= matched
+
+ if $CODERAY_DEBUG and not kind
+ raise_inspect 'Error token %p in line %d' %
+ [[match, kind], line], tokens, state
+ end
+ raise_inspect 'Empty token', tokens, state unless match
+
+ tokens << [match, kind]
+
+ end
+
+ tokens
+ end
+
+ end
+
+end
+end
--- /dev/null
+module CodeRay
+
+ # This module holds the Style class and its subclasses.
+ #
+ # See Plugin.
+ module Styles
+ extend PluginHost
+ plugin_path File.dirname(__FILE__), 'styles'
+
+ class Style
+ extend Plugin
+ plugin_host Styles
+
+ DEFAULT_OPTIONS = { }
+
+ end
+
+ end
+
+end
--- /dev/null
+module CodeRay
+module Styles
+
+ default :cycnus
+
+end
+end
--- /dev/null
+module CodeRay
+module Styles
+
+ class Cycnus < Style
+
+ register_for :cycnus
+
+ code_background = '#f8f8f8'
+ numbers_background = '#def'
+ border_color = 'silver'
+ normal_color = '#000'
+
+ CSS_MAIN_STYLES = <<-MAIN
+.CodeRay {
+ background-color: #{code_background};
+ border: 1px solid #{border_color};
+ font-family: 'Courier New', 'Terminal', monospace;
+ color: #{normal_color};
+}
+.CodeRay pre { margin: 0px }
+
+div.CodeRay { }
+
+span.CodeRay { white-space: pre; border: 0px; padding: 2px }
+
+table.CodeRay { border-collapse: collapse; width: 100%; padding: 2px }
+table.CodeRay td { padding: 2px 4px; vertical-align: top }
+
+.CodeRay .line_numbers, .CodeRay .no {
+ background-color: #{numbers_background};
+ color: gray;
+ text-align: right;
+}
+.CodeRay .line_numbers tt { font-weight: bold }
+.CodeRay .line_numbers .highlighted { color: red }
+.CodeRay .line { display: block; float: left; width: 100%; }
+.CodeRay .no { padding: 0px 4px }
+.CodeRay .code { width: 100% }
+
+ol.CodeRay { font-size: 10pt }
+ol.CodeRay li { white-space: pre }
+
+.CodeRay .code pre { overflow: auto }
+ MAIN
+
+ TOKEN_COLORS = <<-'TOKENS'
+.debug { color:white ! important; background:blue ! important; }
+
+.af { color:#00C }
+.an { color:#007 }
+.at { color:#f08 }
+.av { color:#700 }
+.aw { color:#C00 }
+.bi { color:#509; font-weight:bold }
+.c { color:#888; }
+
+.ch { color:#04D }
+.ch .k { color:#04D }
+.ch .dl { color:#039 }
+
+.cl { color:#B06; font-weight:bold }
+.cm { color:#A08; font-weight:bold }
+.co { color:#036; font-weight:bold }
+.cr { color:#0A0 }
+.cv { color:#369 }
+.de { color:#B0B; }
+.df { color:#099; font-weight:bold }
+.di { color:#088; font-weight:bold }
+.dl { color:black }
+.do { color:#970 }
+.dt { color:#34b }
+.ds { color:#D42; font-weight:bold }
+.e { color:#666; font-weight:bold }
+.en { color:#800; font-weight:bold }
+.er { color:#F00; background-color:#FAA }
+.ex { color:#C00; font-weight:bold }
+.fl { color:#60E; font-weight:bold }
+.fu { color:#06B; font-weight:bold }
+.gv { color:#d70; font-weight:bold }
+.hx { color:#058; font-weight:bold }
+.i { color:#00D; font-weight:bold }
+.ic { color:#B44; font-weight:bold }
+
+.il { background: #ddd; color: black }
+.il .il { background: #ccc }
+.il .il .il { background: #bbb }
+.il .idl { background: #ddd; font-weight: bold; color: #666 }
+.idl { background-color: #bbb; font-weight: bold; color: #666; }
+
+.im { color:#f00; }
+.in { color:#B2B; font-weight:bold }
+.iv { color:#33B }
+.la { color:#970; font-weight:bold }
+.lv { color:#963 }
+.oc { color:#40E; font-weight:bold }
+.of { color:#000; font-weight:bold }
+.op { }
+.pc { color:#038; font-weight:bold }
+.pd { color:#369; font-weight:bold }
+.pp { color:#579; }
+.ps { color:#00C; font-weight:bold }
+.pt { color:#074; font-weight:bold }
+.r, .kw { color:#080; font-weight:bold }
+
+.ke { color: #808; }
+.ke .dl { color: #606; }
+.ke .ch { color: #80f; }
+.vl { color: #088; }
+
+.rx { background-color:#fff0ff }
+.rx .k { color:#808 }
+.rx .dl { color:#404 }
+.rx .mod { color:#C2C }
+.rx .fu { color:#404; font-weight: bold }
+
+.s { background-color:#fff0f0; color: #D20; }
+.s .s { background-color:#ffe0e0 }
+.s .s .s { background-color:#ffd0d0 }
+.s .k { }
+.s .ch { color: #b0b; }
+.s .dl { color: #710; }
+
+.sh { background-color:#f0fff0; color:#2B2 }
+.sh .k { }
+.sh .dl { color:#161 }
+
+.sy { color:#A60 }
+.sy .k { color:#A60 }
+.sy .dl { color:#630 }
+
+.ta { color:#070 }
+.tf { color:#070; font-weight:bold }
+.ts { color:#D70; font-weight:bold }
+.ty { color:#339; font-weight:bold }
+.v { color:#036 }
+.xt { color:#444 }
+
+.ins { background: #afa; }
+.del { background: #faa; }
+.chg { color: #aaf; background: #007; }
+.head { color: #f8f; background: #505 }
+
+.ins .ins { color: #080; font-weight:bold }
+.del .del { color: #800; font-weight:bold }
+.chg .chg { color: #66f; }
+.head .head { color: #f4f; }
+ TOKENS
+
+ end
+
+end
+end
--- /dev/null
+module CodeRay
+module Styles
+
+ class Murphy < Style
+
+ register_for :murphy
+
+ code_background = '#001129'
+ numbers_background = code_background
+ border_color = 'silver'
+ normal_color = '#C0C0C0'
+
+ CSS_MAIN_STYLES = <<-MAIN
+.CodeRay {
+ background-color: #{code_background};
+ border: 1px solid #{border_color};
+ font-family: 'Courier New', 'Terminal', monospace;
+ color: #{normal_color};
+}
+.CodeRay pre { margin: 0px; }
+
+div.CodeRay { }
+
+span.CodeRay { white-space: pre; border: 0px; padding: 2px; }
+
+table.CodeRay { border-collapse: collapse; width: 100%; padding: 2px; }
+table.CodeRay td { padding: 2px 4px; vertical-align: top; }
+
+.CodeRay .line_numbers, .CodeRay .no {
+ background-color: #{numbers_background};
+ color: gray;
+ text-align: right;
+}
+.CodeRay .line_numbers tt { font-weight: bold; }
+.CodeRay .line_numbers .highlighted { color: red }
+.CodeRay .line { display: block; float: left; width: 100%; }
+.CodeRay .no { padding: 0px 4px; }
+.CodeRay .code { width: 100%; }
+
+ol.CodeRay { font-size: 10pt; }
+ol.CodeRay li { white-space: pre; }
+
+.CodeRay .code pre { overflow: auto; }
+ MAIN
+
+ TOKEN_COLORS = <<-'TOKENS'
+.af { color:#00C; }
+.an { color:#007; }
+.av { color:#700; }
+.aw { color:#C00; }
+.bi { color:#509; font-weight:bold; }
+.c { color:#555; background-color: black; }
+
+.ch { color:#88F; }
+.ch .k { color:#04D; }
+.ch .dl { color:#039; }
+
+.cl { color:#e9e; font-weight:bold; }
+.co { color:#5ED; font-weight:bold; }
+.cr { color:#0A0; }
+.cv { color:#ccf; }
+.df { color:#099; font-weight:bold; }
+.di { color:#088; font-weight:bold; }
+.dl { color:black; }
+.do { color:#970; }
+.ds { color:#D42; font-weight:bold; }
+.e { color:#666; font-weight:bold; }
+.er { color:#F00; background-color:#FAA; }
+.ex { color:#F00; font-weight:bold; }
+.fl { color:#60E; font-weight:bold; }
+.fu { color:#5ed; font-weight:bold; }
+.gv { color:#f84; }
+.hx { color:#058; font-weight:bold; }
+.i { color:#66f; font-weight:bold; }
+.ic { color:#B44; font-weight:bold; }
+.il { }
+.in { color:#B2B; font-weight:bold; }
+.iv { color:#aaf; }
+.la { color:#970; font-weight:bold; }
+.lv { color:#963; }
+.oc { color:#40E; font-weight:bold; }
+.of { color:#000; font-weight:bold; }
+.op { }
+.pc { color:#08f; font-weight:bold; }
+.pd { color:#369; font-weight:bold; }
+.pp { color:#579; }
+.pt { color:#66f; font-weight:bold; }
+.r { color:#5de; font-weight:bold; }
+.r, .kw { color:#5de; font-weight:bold }
+
+.ke { color: #808; }
+
+.rx { background-color:#221133; }
+.rx .k { color:#f8f; }
+.rx .dl { color:#f0f; }
+.rx .mod { color:#f0b; }
+.rx .fu { color:#404; font-weight: bold; }
+
+.s { background-color:#331122; }
+.s .s { background-color:#ffe0e0; }
+.s .s .s { background-color:#ffd0d0; }
+.s .k { color:#F88; }
+.s .dl { color:#f55; }
+
+.sh { background-color:#f0fff0; }
+.sh .k { color:#2B2; }
+.sh .dl { color:#161; }
+
+.sy { color:#Fc8; }
+.sy .k { color:#Fc8; }
+.sy .dl { color:#F84; }
+
+.ta { color:#070; }
+.tf { color:#070; font-weight:bold; }
+.ts { color:#D70; font-weight:bold; }
+.ty { color:#339; font-weight:bold; }
+.v { color:#036; }
+.xt { color:#444; }
+
+.ins { background: #afa; }
+.del { background: #faa; }
+.chg { color: #aaf; background: #007; }
+.head { color: #f8f; background: #505 }
+
+.ins .ins { color: #080; font-weight:bold }
+.del .del { color: #800; font-weight:bold }
+.chg .chg { color: #66f; }
+.head .head { color: #f4f; }
+ TOKENS
+
+ end
+
+end
+end
--- /dev/null
+module CodeRay
+ class Tokens
+ ClassOfKind = Hash.new do |h, k|
+ h[k] = k.to_s
+ end
+ ClassOfKind.update with = {
+ :annotation => 'at',
+ :attribute_name => 'an',
+ :attribute_name_fat => 'af',
+ :attribute_value => 'av',
+ :attribute_value_fat => 'aw',
+ :bin => 'bi',
+ :char => 'ch',
+ :class => 'cl',
+ :class_variable => 'cv',
+ :color => 'cr',
+ :comment => 'c',
+ :complex => 'cm',
+ :constant => 'co',
+ :content => 'k',
+ :decorator => 'de',
+ :definition => 'df',
+ :delimiter => 'dl',
+ :directive => 'di',
+ :doc => 'do',
+ :doctype => 'dt',
+ :doc_string => 'ds',
+ :entity => 'en',
+ :error => 'er',
+ :escape => 'e',
+ :exception => 'ex',
+ :float => 'fl',
+ :function => 'fu',
+ :global_variable => 'gv',
+ :hex => 'hx',
+ :imaginary => 'cm',
+ :important => 'im',
+ :include => 'ic',
+ :inline => 'il',
+ :inline_delimiter => 'idl',
+ :instance_variable => 'iv',
+ :integer => 'i',
+ :interpreted => 'in',
+ :keyword => 'kw',
+ :key => 'ke',
+ :label => 'la',
+ :local_variable => 'lv',
+ :modifier => 'mod',
+ :oct => 'oc',
+ :operator_fat => 'of',
+ :pre_constant => 'pc',
+ :pre_type => 'pt',
+ :predefined => 'pd',
+ :preprocessor => 'pp',
+ :pseudo_class => 'ps',
+ :regexp => 'rx',
+ :reserved => 'r',
+ :shell => 'sh',
+ :string => 's',
+ :symbol => 'sy',
+ :tag => 'ta',
+ :tag_fat => 'tf',
+ :tag_special => 'ts',
+ :type => 'ty',
+ :variable => 'v',
+ :value => 'vl',
+ :xml_text => 'xt',
+
+ :insert => 'ins',
+ :delete => 'del',
+ :change => 'chg',
+ :head => 'head',
+
+ :ident => :NO_HIGHLIGHT, # 'id'
+ #:operator => 'op',
+ :operator => :NO_HIGHLIGHT, # 'op'
+ :space => :NO_HIGHLIGHT, # 'sp'
+ :plain => :NO_HIGHLIGHT,
+ }
+ ClassOfKind[:method] = ClassOfKind[:function]
+ ClassOfKind[:open] = ClassOfKind[:close] = ClassOfKind[:delimiter]
+ ClassOfKind[:nesting_delimiter] = ClassOfKind[:delimiter]
+ ClassOfKind[:escape] = ClassOfKind[:delimiter]
+ #ClassOfKind.default = ClassOfKind[:error] or raise 'no class found for :error!'
+ end
+end
\ No newline at end of file
--- /dev/null
+module CodeRay
+
+ # = Tokens
+ #
+ # The Tokens class represents a list of tokens returnd from
+ # a Scanner.
+ #
+ # A token is not a special object, just a two-element Array
+ # consisting of
+ # * the _token_ _text_ (the original source of the token in a String) or
+ # a _token_ _action_ (:open, :close, :begin_line, :end_line)
+ # * the _token_ _kind_ (a Symbol representing the type of the token)
+ #
+ # A token looks like this:
+ #
+ # ['# It looks like this', :comment]
+ # ['3.1415926', :float]
+ # ['$^', :error]
+ #
+ # Some scanners also yield sub-tokens, represented by special
+ # token actions, namely :open and :close.
+ #
+ # The Ruby scanner, for example, splits "a string" into:
+ #
+ # [
+ # [:open, :string],
+ # ['"', :delimiter],
+ # ['a string', :content],
+ # ['"', :delimiter],
+ # [:close, :string]
+ # ]
+ #
+ # Tokens is the interface between Scanners and Encoders:
+ # The input is split and saved into a Tokens object. The Encoder
+ # then builds the output from this object.
+ #
+ # Thus, the syntax below becomes clear:
+ #
+ # CodeRay.scan('price = 2.59', :ruby).html
+ # # the Tokens object is here -------^
+ #
+ # See how small it is? ;)
+ #
+ # Tokens gives you the power to handle pre-scanned code very easily:
+ # You can convert it to a webpage, a YAML file, or dump it into a gzip'ed string
+ # that you put in your DB.
+ #
+ # It also allows you to generate tokens directly (without using a scanner),
+ # to load them from a file, and still use any Encoder that CodeRay provides.
+ #
+ # Tokens' subclass TokenStream allows streaming to save memory.
+ class Tokens < Array
+
+ # The Scanner instance that created the tokens.
+ attr_accessor :scanner
+
+ # Whether the object is a TokenStream.
+ #
+ # Returns false.
+ def stream?
+ false
+ end
+
+ # Iterates over all tokens.
+ #
+ # If a filter is given, only tokens of that kind are yielded.
+ def each kind_filter = nil, &block
+ unless kind_filter
+ super(&block)
+ else
+ super() do |text, kind|
+ next unless kind == kind_filter
+ yield text, kind
+ end
+ end
+ end
+
+ # Iterates over all text tokens.
+ # Range tokens like [:open, :string] are left out.
+ #
+ # Example:
+ # tokens.each_text_token { |text, kind| text.replace html_escape(text) }
+ def each_text_token
+ each do |text, kind|
+ next unless text.is_a? ::String
+ yield text, kind
+ end
+ end
+
+ # Encode the tokens using encoder.
+ #
+ # encoder can be
+ # * a symbol like :html oder :statistic
+ # * an Encoder class
+ # * an Encoder object
+ #
+ # options are passed to the encoder.
+ def encode encoder, options = {}
+ unless encoder.is_a? Encoders::Encoder
+ unless encoder.is_a? Class
+ encoder_class = Encoders[encoder]
+ end
+ encoder = encoder_class.new options
+ end
+ encoder.encode_tokens self, options
+ end
+
+
+ # Turn into a string using Encoders::Text.
+ #
+ # +options+ are passed to the encoder if given.
+ def to_s options = {}
+ encode :text, options
+ end
+
+ # Redirects unknown methods to encoder calls.
+ #
+ # For example, if you call +tokens.html+, the HTML encoder
+ # is used to highlight the tokens.
+ def method_missing meth, options = {}
+ Encoders[meth].new(options).encode_tokens self
+ end
+
+ # Returns the tokens compressed by joining consecutive
+ # tokens of the same kind.
+ #
+ # This can not be undone, but should yield the same output
+ # in most Encoders. It basically makes the output smaller.
+ #
+ # Combined with dump, it saves space for the cost of time.
+ #
+ # If the scanner is written carefully, this is not required -
+ # for example, consecutive //-comment lines could already be
+ # joined in one comment token by the Scanner.
+ def optimize
+ last_kind = last_text = nil
+ new = self.class.new
+ for text, kind in self
+ if text.is_a? String
+ if kind == last_kind
+ last_text << text
+ else
+ new << [last_text, last_kind] if last_kind
+ last_text = text
+ last_kind = kind
+ end
+ else
+ new << [last_text, last_kind] if last_kind
+ last_kind = last_text = nil
+ new << [text, kind]
+ end
+ end
+ new << [last_text, last_kind] if last_kind
+ new
+ end
+
+ # Compact the object itself; see optimize.
+ def optimize!
+ replace optimize
+ end
+
+ # Ensure that all :open tokens have a correspondent :close one.
+ #
+ # TODO: Test this!
+ def fix
+ tokens = self.class.new
+ # Check token nesting using a stack of kinds.
+ opened = []
+ for type, kind in self
+ case type
+ when :open
+ opened.push [:close, kind]
+ when :begin_line
+ opened.push [:end_line, kind]
+ when :close, :end_line
+ expected = opened.pop
+ if [type, kind] != expected
+ # Unexpected :close; decide what to do based on the kind:
+ # - token was never opened: delete the :close (just skip it)
+ next unless opened.rindex expected
+ # - token was opened earlier: also close tokens in between
+ tokens << token until (token = opened.pop) == expected
+ end
+ end
+ tokens << [type, kind]
+ end
+ # Close remaining opened tokens
+ tokens << token while token = opened.pop
+ tokens
+ end
+
+ def fix!
+ replace fix
+ end
+
+ # TODO: Scanner#split_into_lines
+ #
+ # Makes sure that:
+ # - newlines are single tokens
+ # (which means all other token are single-line)
+ # - there are no open tokens at the end the line
+ #
+ # This makes it simple for encoders that work line-oriented,
+ # like HTML with list-style numeration.
+ def split_into_lines
+ raise NotImplementedError
+ end
+
+ def split_into_lines!
+ replace split_into_lines
+ end
+
+ # Dumps the object into a String that can be saved
+ # in files or databases.
+ #
+ # The dump is created with Marshal.dump;
+ # In addition, it is gzipped using GZip.gzip.
+ #
+ # The returned String object includes Undumping
+ # so it has an #undump method. See Tokens.load.
+ #
+ # You can configure the level of compression,
+ # but the default value 7 should be what you want
+ # in most cases as it is a good compromise between
+ # speed and compression rate.
+ #
+ # See GZip module.
+ def dump gzip_level = 7
+ require 'coderay/helpers/gzip_simple'
+ dump = Marshal.dump self
+ dump = dump.gzip gzip_level
+ dump.extend Undumping
+ end
+
+ # The total size of the tokens.
+ # Should be equal to the input size before
+ # scanning.
+ def text_size
+ size = 0
+ each_text_token do |t, k|
+ size + t.size
+ end
+ size
+ end
+
+ # Return all text tokens joined into a single string.
+ def text
+ map { |t, k| t if t.is_a? ::String }.join
+ end
+
+ # Include this module to give an object an #undump
+ # method.
+ #
+ # The string returned by Tokens.dump includes Undumping.
+ module Undumping
+ # Calls Tokens.load with itself.
+ def undump
+ Tokens.load self
+ end
+ end
+
+ # Undump the object using Marshal.load, then
+ # unzip it using GZip.gunzip.
+ #
+ # The result is commonly a Tokens object, but
+ # this is not guaranteed.
+ def Tokens.load dump
+ require 'coderay/helpers/gzip_simple'
+ dump = dump.gunzip
+ @dump = Marshal.load dump
+ end
+
+ end
+
+
+ # = TokenStream
+ #
+ # The TokenStream class is a fake Array without elements.
+ #
+ # It redirects the method << to a block given at creation.
+ #
+ # This allows scanners and Encoders to use streaming (no
+ # tokens are saved, the input is highlighted the same time it
+ # is scanned) with the same code.
+ #
+ # See CodeRay.encode_stream and CodeRay.scan_stream
+ class TokenStream < Tokens
+
+ # Whether the object is a TokenStream.
+ #
+ # Returns true.
+ def stream?
+ true
+ end
+
+ # The Array is empty, but size counts the tokens given by <<.
+ attr_reader :size
+
+ # Creates a new TokenStream that calls +block+ whenever
+ # its << method is called.
+ #
+ # Example:
+ #
+ # require 'coderay'
+ #
+ # token_stream = CodeRay::TokenStream.new do |text, kind|
+ # puts 'kind: %s, text size: %d.' % [kind, text.size]
+ # end
+ #
+ # token_stream << ['/\d+/', :regexp]
+ # #-> kind: rexpexp, text size: 5.
+ #
+ def initialize &block
+ raise ArgumentError, 'Block expected for streaming.' unless block
+ @callback = block
+ @size = 0
+ end
+
+ # Calls +block+ with +token+ and increments size.
+ #
+ # Returns self.
+ def << token
+ @callback.call(*token)
+ @size += 1
+ self
+ end
+
+ # This method is not implemented due to speed reasons. Use Tokens.
+ def text_size
+ raise NotImplementedError,
+ 'This method is not implemented due to speed reasons.'
+ end
+
+ # A TokenStream cannot be dumped. Use Tokens.
+ def dump
+ raise NotImplementedError, 'A TokenStream cannot be dumped.'
+ end
+
+ # A TokenStream cannot be optimized. Use Tokens.
+ def optimize
+ raise NotImplementedError, 'A TokenStream cannot be optimized.'
+ end
+
+ end
+
+end
+
+if $0 == __FILE__
+ $VERBOSE = true
+ $: << File.join(File.dirname(__FILE__), '..')
+ eval DATA.read, nil, $0, __LINE__ + 4
+end
+
+__END__
+require 'test/unit'
+
+class TokensTest < Test::Unit::TestCase
+
+ def test_creation
+ assert CodeRay::Tokens < Array
+ tokens = nil
+ assert_nothing_raised do
+ tokens = CodeRay::Tokens.new
+ end
+ assert_kind_of Array, tokens
+ end
+
+ def test_adding_tokens
+ tokens = CodeRay::Tokens.new
+ assert_nothing_raised do
+ tokens << ['string', :type]
+ tokens << ['()', :operator]
+ end
+ assert_equal tokens.size, 2
+ end
+
+ def test_dump_undump
+ tokens = CodeRay::Tokens.new
+ assert_nothing_raised do
+ tokens << ['string', :type]
+ tokens << ['()', :operator]
+ end
+ tokens2 = nil
+ assert_nothing_raised do
+ tokens2 = tokens.dump.undump
+ end
+ assert_equal tokens, tokens2
+ end
+
+end
\ No newline at end of file
--- /dev/null
+require 'test/unit'
+require 'coderay'
+
+class BasicTest < Test::Unit::TestCase
+
+ def test_version
+ assert_nothing_raised do
+ assert_match(/\A\d\.\d\.\d\z/, CodeRay::VERSION)
+ end
+ end
+
+ RUBY_TEST_CODE = 'puts "Hello, World!"'
+
+ RUBY_TEST_TOKENS = [
+ ['puts', :ident],
+ [' ', :space],
+ [:open, :string],
+ ['"', :delimiter],
+ ['Hello, World!', :content],
+ ['"', :delimiter],
+ [:close, :string]
+ ]
+ def test_simple_scan
+ assert_nothing_raised do
+ assert_equal RUBY_TEST_TOKENS, CodeRay.scan(RUBY_TEST_CODE, :ruby).to_ary
+ end
+ end
+
+ RUBY_TEST_HTML = 'puts <span class="s"><span class="dl">"</span>' +
+ '<span class="k">Hello, World!</span><span class="dl">"</span></span>'
+ def test_simple_highlight
+ assert_nothing_raised do
+ assert_equal RUBY_TEST_HTML, CodeRay.scan(RUBY_TEST_CODE, :ruby).html
+ end
+ end
+
+ def test_duo
+ assert_equal(RUBY_TEST_CODE,
+ CodeRay::Duo[:plain, :plain].highlight(RUBY_TEST_CODE))
+ assert_equal(RUBY_TEST_CODE,
+ CodeRay::Duo[:plain => :plain].highlight(RUBY_TEST_CODE))
+ end
+
+ def test_duo_stream
+ assert_equal(RUBY_TEST_CODE,
+ CodeRay::Duo[:plain, :plain].highlight(RUBY_TEST_CODE, :stream => true))
+ end
+
+ def test_comment_filter
+ assert_equal <<-EXPECTED, CodeRay.scan(<<-INPUT, :ruby).comment_filter.text
+#!/usr/bin/env ruby
+
+code
+
+more code
+ EXPECTED
+#!/usr/bin/env ruby
+=begin
+A multi-line comment.
+=end
+code
+# A single-line comment.
+more code # and another comment, in-line.
+ INPUT
+ end
+
+ def test_lines_of_code
+ assert_equal 2, CodeRay.scan(<<-INPUT, :ruby).lines_of_code
+#!/usr/bin/env ruby
+=begin
+A multi-line comment.
+=end
+code
+# A single-line comment.
+more code # and another comment, in-line.
+ INPUT
+ rHTML = <<-RHTML
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
+ "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
+
+<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
+<head>
+ <meta http-equiv="content-type" content="text/html;charset=UTF-8" />
+ <title><%= controller.controller_name.titleize %>: <%= controller.action_name %></title>
+ <%= stylesheet_link_tag 'scaffold' %>
+</head>
+<body>
+
+<p style="color: green"><%= flash[:notice] %></p>
+
+<div id="main">
+ <%= yield %>
+</div>
+
+</body>
+</html>
+ RHTML
+ assert_equal 0, CodeRay.scan(rHTML, :html).lines_of_code
+ assert_equal 0, CodeRay.scan(rHTML, :php).lines_of_code
+ assert_equal 0, CodeRay.scan(rHTML, :yaml).lines_of_code
+ assert_equal 4, CodeRay.scan(rHTML, :rhtml).lines_of_code
+ end
+
+ def test_rubygems_not_loaded
+ assert_equal nil, defined? Gem
+ end if ENV['check_rubygems'] && RUBY_VERSION < '1.9'
+
+ def test_list_of_encoders
+ assert_kind_of(Array, CodeRay::Encoders.list)
+ assert CodeRay::Encoders.list.include?('count')
+ end
+
+ def test_list_of_scanners
+ assert_kind_of(Array, CodeRay::Scanners.list)
+ assert CodeRay::Scanners.list.include?('plaintext')
+ end
+
+ def test_scan_a_frozen_string
+ CodeRay.scan RUBY_VERSION, :ruby
+ end
+
+end
--- /dev/null
+!RBIX
+0
+x
+M
+1
+n
+n
+x
+10
+__script__
+i
+53
+5
+7
+0
+64
+47
+49
+1
+1
+15
+5
+7
+2
+64
+47
+49
+1
+1
+15
+99
+7
+3
+45
+4
+5
+43
+6
+43
+7
+65
+49
+8
+3
+13
+99
+12
+7
+9
+12
+7
+10
+12
+65
+12
+49
+11
+4
+15
+49
+9
+0
+15
+2
+11
+I
+6
+I
+0
+I
+0
+I
+0
+n
+p
+12
+s
+9
+test/unit
+x
+7
+require
+s
+7
+coderay
+x
+9
+BasicTest
+x
+4
+Test
+n
+x
+4
+Unit
+x
+8
+TestCase
+x
+10
+open_class
+x
+14
+__class_init__
+M
+1
+n
+n
+x
+9
+BasicTest
+i
+263
+5
+66
+99
+7
+0
+7
+1
+65
+67
+49
+2
+0
+49
+3
+4
+15
+65
+7
+4
+7
+5
+64
+49
+6
+2
+15
+65
+7
+7
+7
+8
+64
+7
+9
+35
+2
+7
+10
+64
+7
+11
+35
+2
+7
+12
+7
+13
+35
+2
+7
+14
+64
+7
+15
+35
+2
+7
+16
+64
+7
+17
+35
+2
+7
+14
+64
+7
+15
+35
+2
+7
+18
+7
+13
+35
+2
+35
+7
+49
+6
+2
+15
+99
+7
+19
+7
+20
+65
+67
+49
+2
+0
+49
+3
+4
+15
+65
+7
+21
+7
+22
+64
+7
+23
+64
+81
+24
+49
+6
+2
+15
+99
+7
+25
+7
+26
+65
+67
+49
+2
+0
+49
+3
+4
+15
+99
+7
+27
+7
+28
+65
+67
+49
+2
+0
+49
+3
+4
+15
+99
+7
+29
+7
+30
+65
+67
+49
+2
+0
+49
+3
+4
+15
+99
+7
+31
+7
+32
+65
+67
+49
+2
+0
+49
+3
+4
+15
+99
+7
+33
+7
+34
+65
+67
+49
+2
+0
+49
+3
+4
+15
+45
+35
+36
+7
+37
+64
+49
+38
+1
+13
+9
+202
+15
+45
+39
+40
+7
+41
+64
+84
+42
+9
+219
+99
+7
+43
+7
+44
+65
+67
+49
+2
+0
+49
+3
+4
+8
+220
+1
+15
+99
+7
+45
+7
+46
+65
+67
+49
+2
+0
+49
+3
+4
+15
+99
+7
+47
+7
+48
+65
+67
+49
+2
+0
+49
+3
+4
+15
+99
+7
+49
+7
+50
+65
+67
+49
+2
+0
+49
+3
+4
+11
+I
+a
+I
+0
+I
+0
+I
+0
+n
+p
+51
+x
+12
+test_version
+M
+1
+n
+n
+x
+12
+test_version
+i
+8
+5
+56
+0
+47
+50
+1
+0
+11
+I
+2
+I
+0
+I
+0
+I
+0
+n
+p
+2
+M
+1
+p
+2
+x
+9
+for_block
+t
+n
+x
+12
+test_version
+i
+29
+5
+7
+0
+13
+70
+9
+19
+15
+44
+43
+1
+7
+2
+78
+49
+3
+2
+6
+0
+45
+4
+5
+43
+6
+47
+49
+7
+2
+11
+I
+5
+I
+0
+I
+0
+I
+0
+I
+-2
+p
+8
+n
+x
+6
+Regexp
+s
+14
+\A\d\.\d\.\d\z
+x
+3
+new
+x
+7
+CodeRay
+n
+x
+7
+VERSION
+x
+12
+assert_match
+p
+5
+I
+0
+I
+7
+I
+0
+I
+8
+I
+1d
+x
+55
+/Users/murphy/ruby/coderay-0.9/test/functional/basic.rb
+p
+0
+x
+21
+assert_nothing_raised
+p
+5
+I
+0
+I
+6
+I
+0
+I
+7
+I
+8
+x
+55
+/Users/murphy/ruby/coderay-0.9/test/functional/basic.rb
+p
+0
+x
+17
+method_visibility
+x
+15
+add_defn_method
+x
+14
+RUBY_TEST_CODE
+s
+20
+puts "Hello, World!"
+x
+9
+const_set
+x
+16
+RUBY_TEST_TOKENS
+s
+4
+puts
+x
+5
+ident
+s
+1
+
+x
+5
+space
+x
+4
+open
+x
+6
+string
+s
+1
+"
+x
+9
+delimiter
+s
+13
+Hello, World!
+x
+7
+content
+x
+5
+close
+x
+16
+test_simple_scan
+M
+1
+n
+n
+x
+16
+test_simple_scan
+i
+8
+5
+56
+0
+47
+50
+1
+0
+11
+I
+2
+I
+0
+I
+0
+I
+0
+n
+p
+2
+M
+1
+p
+2
+x
+9
+for_block
+t
+n
+x
+16
+test_simple_scan
+i
+23
+5
+45
+0
+1
+45
+2
+3
+45
+4
+5
+7
+6
+49
+7
+2
+49
+8
+0
+47
+49
+9
+2
+11
+I
+6
+I
+0
+I
+0
+I
+0
+I
+-2
+p
+10
+x
+16
+RUBY_TEST_TOKENS
+n
+x
+7
+CodeRay
+n
+x
+14
+RUBY_TEST_CODE
+n
+x
+4
+ruby
+x
+4
+scan
+x
+6
+to_ary
+x
+12
+assert_equal
+p
+5
+I
+0
+I
+18
+I
+0
+I
+19
+I
+17
+x
+55
+/Users/murphy/ruby/coderay-0.9/test/functional/basic.rb
+p
+0
+x
+21
+assert_nothing_raised
+p
+5
+I
+0
+I
+17
+I
+0
+I
+18
+I
+8
+x
+55
+/Users/murphy/ruby/coderay-0.9/test/functional/basic.rb
+p
+0
+x
+14
+RUBY_TEST_HTML
+s
+51
+puts <span class="s"><span class="dl">"</span>
+s
+73
+<span class="k">Hello, World!</span><span class="dl">"</span></span>
+x
+1
++
+x
+21
+test_simple_highlight
+M
+1
+n
+n
+x
+21
+test_simple_highlight
+i
+8
+5
+56
+0
+47
+50
+1
+0
+11
+I
+2
+I
+0
+I
+0
+I
+0
+n
+p
+2
+M
+1
+p
+2
+x
+9
+for_block
+t
+n
+x
+21
+test_simple_highlight
+i
+23
+5
+45
+0
+1
+45
+2
+3
+45
+4
+5
+7
+6
+49
+7
+2
+49
+8
+0
+47
+49
+9
+2
+11
+I
+6
+I
+0
+I
+0
+I
+0
+I
+-2
+p
+10
+x
+14
+RUBY_TEST_HTML
+n
+x
+7
+CodeRay
+n
+x
+14
+RUBY_TEST_CODE
+n
+x
+4
+ruby
+x
+4
+scan
+x
+4
+html
+x
+12
+assert_equal
+p
+5
+I
+0
+I
+20
+I
+0
+I
+21
+I
+17
+x
+55
+/Users/murphy/ruby/coderay-0.9/test/functional/basic.rb
+p
+0
+x
+21
+assert_nothing_raised
+p
+5
+I
+0
+I
+1f
+I
+0
+I
+20
+I
+8
+x
+55
+/Users/murphy/ruby/coderay-0.9/test/functional/basic.rb
+p
+0
+x
+8
+test_duo
+M
+1
+n
+n
+x
+8
+test_duo
+i
+66
+5
+45
+0
+1
+45
+2
+3
+43
+4
+7
+5
+7
+5
+49
+6
+2
+45
+0
+7
+49
+8
+1
+47
+49
+9
+2
+15
+5
+45
+0
+10
+45
+2
+11
+43
+4
+44
+43
+12
+79
+49
+13
+1
+13
+7
+5
+7
+5
+49
+14
+2
+15
+49
+6
+1
+45
+0
+15
+49
+8
+1
+47
+49
+9
+2
+11
+I
+7
+I
+0
+I
+0
+I
+0
+n
+p
+16
+x
+14
+RUBY_TEST_CODE
+n
+x
+7
+CodeRay
+n
+x
+3
+Duo
+x
+5
+plain
+x
+2
+[]
+n
+x
+9
+highlight
+x
+12
+assert_equal
+n
+n
+x
+4
+Hash
+x
+16
+new_from_literal
+x
+3
+[]=
+n
+p
+15
+I
+0
+I
+25
+I
+0
+I
+26
+I
+4
+I
+27
+I
+16
+I
+26
+I
+1b
+I
+28
+I
+1f
+I
+29
+I
+3d
+I
+28
+I
+42
+x
+55
+/Users/murphy/ruby/coderay-0.9/test/functional/basic.rb
+p
+0
+x
+15
+test_duo_stream
+M
+1
+n
+n
+x
+15
+test_duo_stream
+i
+42
+5
+45
+0
+1
+45
+2
+3
+43
+4
+7
+5
+7
+5
+49
+6
+2
+45
+0
+7
+44
+43
+8
+79
+49
+9
+1
+13
+7
+10
+2
+49
+11
+2
+15
+49
+12
+2
+47
+49
+13
+2
+11
+I
+8
+I
+0
+I
+0
+I
+0
+n
+p
+14
+x
+14
+RUBY_TEST_CODE
+n
+x
+7
+CodeRay
+n
+x
+3
+Duo
+x
+5
+plain
+x
+2
+[]
+n
+x
+4
+Hash
+x
+16
+new_from_literal
+x
+6
+stream
+x
+3
+[]=
+x
+9
+highlight
+x
+12
+assert_equal
+p
+9
+I
+0
+I
+2c
+I
+0
+I
+2d
+I
+4
+I
+2e
+I
+25
+I
+2d
+I
+2a
+x
+55
+/Users/murphy/ruby/coderay-0.9/test/functional/basic.rb
+p
+0
+x
+19
+test_comment_filter
+M
+1
+n
+n
+x
+19
+test_comment_filter
+i
+26
+5
+7
+0
+64
+45
+1
+2
+7
+3
+64
+7
+4
+49
+5
+2
+49
+6
+0
+49
+7
+0
+47
+49
+8
+2
+11
+I
+5
+I
+0
+I
+0
+I
+0
+n
+p
+9
+s
+39
+#!/usr/bin/env ruby
+
+code
+
+more code
+
+x
+7
+CodeRay
+n
+s
+127
+#!/usr/bin/env ruby
+=begin
+A multi-line comment.
+=end
+code
+# A single-line comment.
+more code # and another comment, in-line.
+
+x
+4
+ruby
+x
+4
+scan
+x
+14
+comment_filter
+x
+4
+text
+x
+12
+assert_equal
+p
+5
+I
+0
+I
+31
+I
+0
+I
+32
+I
+1a
+x
+55
+/Users/murphy/ruby/coderay-0.9/test/functional/basic.rb
+p
+0
+x
+18
+test_lines_of_code
+M
+1
+n
+n
+x
+18
+test_lines_of_code
+i
+108
+5
+80
+45
+0
+1
+7
+2
+64
+7
+3
+49
+4
+2
+49
+5
+0
+47
+49
+6
+2
+15
+7
+7
+64
+19
+0
+15
+5
+78
+45
+0
+8
+20
+0
+7
+9
+49
+4
+2
+49
+5
+0
+47
+49
+6
+2
+15
+5
+78
+45
+0
+10
+20
+0
+7
+11
+49
+4
+2
+49
+5
+0
+47
+49
+6
+2
+15
+5
+78
+45
+0
+12
+20
+0
+7
+13
+49
+4
+2
+49
+5
+0
+47
+49
+6
+2
+15
+5
+4
+4
+45
+0
+14
+20
+0
+7
+15
+49
+4
+2
+49
+5
+0
+47
+49
+6
+2
+11
+I
+6
+I
+1
+I
+0
+I
+0
+n
+p
+16
+x
+7
+CodeRay
+n
+s
+127
+#!/usr/bin/env ruby
+=begin
+A multi-line comment.
+=end
+code
+# A single-line comment.
+more code # and another comment, in-line.
+
+x
+4
+ruby
+x
+4
+scan
+x
+13
+lines_of_code
+x
+12
+assert_equal
+s
+514
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
+ "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
+
+<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
+<head>
+ <meta http-equiv="content-type" content="text/html;charset=UTF-8" />
+ <title><%= controller.controller_name.titleize %>: <%= controller.action_name %></title>
+ <%= stylesheet_link_tag 'scaffold' %>
+</head>
+<body>
+
+<p style="color: green"><%= flash[:notice] %></p>
+
+<div id="main">
+ <%= yield %>
+</div>
+
+</body>
+</html>
+
+n
+x
+4
+html
+n
+x
+3
+php
+n
+x
+4
+yaml
+n
+x
+5
+rhtml
+p
+15
+I
+0
+I
+43
+I
+0
+I
+44
+I
+15
+I
+4d
+I
+1b
+I
+62
+I
+2f
+I
+63
+I
+43
+I
+64
+I
+57
+I
+65
+I
+6c
+x
+55
+/Users/murphy/ruby/coderay-0.9/test/functional/basic.rb
+p
+1
+x
+5
+rHTML
+x
+3
+ENV
+n
+s
+14
+check_rubygems
+x
+2
+[]
+x
+12
+RUBY_VERSION
+n
+s
+3
+1.9
+x
+1
+<
+x
+24
+test_rubygems_not_loaded
+M
+1
+n
+n
+x
+24
+test_rubygems_not_loaded
+i
+34
+5
+1
+26
+93
+0
+15
+29
+17
+0
+7
+0
+98
+1
+1
+30
+8
+23
+25
+92
+0
+27
+8
+28
+15
+7
+2
+8
+29
+1
+47
+49
+3
+2
+11
+I
+4
+I
+0
+I
+0
+I
+0
+n
+p
+4
+x
+3
+Gem
+x
+16
+vm_const_defined
+s
+8
+constant
+x
+12
+assert_equal
+p
+5
+I
+0
+I
+68
+I
+0
+I
+69
+I
+22
+x
+55
+/Users/murphy/ruby/coderay-0.9/test/functional/basic.rb
+p
+0
+x
+21
+test_list_of_encoders
+M
+1
+n
+n
+x
+21
+test_list_of_encoders
+i
+37
+5
+45
+0
+1
+45
+2
+3
+43
+4
+49
+5
+0
+47
+49
+6
+2
+15
+5
+45
+2
+7
+43
+4
+49
+5
+0
+7
+8
+64
+49
+9
+1
+47
+49
+10
+1
+11
+I
+3
+I
+0
+I
+0
+I
+0
+n
+p
+11
+x
+5
+Array
+n
+x
+7
+CodeRay
+n
+x
+8
+Encoders
+x
+4
+list
+x
+14
+assert_kind_of
+n
+s
+5
+count
+x
+8
+include?
+x
+6
+assert
+p
+7
+I
+0
+I
+6c
+I
+0
+I
+6d
+I
+11
+I
+6e
+I
+25
+x
+55
+/Users/murphy/ruby/coderay-0.9/test/functional/basic.rb
+p
+0
+x
+21
+test_list_of_scanners
+M
+1
+n
+n
+x
+21
+test_list_of_scanners
+i
+37
+5
+45
+0
+1
+45
+2
+3
+43
+4
+49
+5
+0
+47
+49
+6
+2
+15
+5
+45
+2
+7
+43
+4
+49
+5
+0
+7
+8
+64
+49
+9
+1
+47
+49
+10
+1
+11
+I
+3
+I
+0
+I
+0
+I
+0
+n
+p
+11
+x
+5
+Array
+n
+x
+7
+CodeRay
+n
+x
+8
+Scanners
+x
+4
+list
+x
+14
+assert_kind_of
+n
+s
+9
+plaintext
+x
+8
+include?
+x
+6
+assert
+p
+7
+I
+0
+I
+71
+I
+0
+I
+72
+I
+11
+I
+73
+I
+25
+x
+55
+/Users/murphy/ruby/coderay-0.9/test/functional/basic.rb
+p
+0
+x
+25
+test_scan_a_frozen_string
+M
+1
+n
+n
+x
+25
+test_scan_a_frozen_string
+i
+12
+45
+0
+1
+45
+2
+3
+7
+4
+49
+5
+2
+11
+I
+3
+I
+0
+I
+0
+I
+0
+n
+p
+6
+x
+7
+CodeRay
+n
+x
+12
+RUBY_VERSION
+n
+x
+4
+ruby
+x
+4
+scan
+p
+5
+I
+0
+I
+76
+I
+0
+I
+77
+I
+c
+x
+55
+/Users/murphy/ruby/coderay-0.9/test/functional/basic.rb
+p
+0
+p
+51
+I
+2
+I
+6
+I
+10
+I
+c
+I
+1a
+I
+e
+I
+1d
+I
+f
+I
+24
+I
+10
+I
+2b
+I
+11
+I
+31
+I
+12
+I
+38
+I
+13
+I
+3f
+I
+14
+I
+46
+I
+15
+I
+52
+I
+17
+I
+60
+I
+1d
+I
+63
+I
+1d
+I
+66
+I
+1e
+I
+6f
+I
+1f
+I
+7d
+I
+25
+I
+8b
+I
+2c
+I
+99
+I
+31
+I
+a7
+I
+43
+I
+b5
+I
+6a
+I
+cc
+I
+68
+I
+db
+I
+6a
+I
+dd
+I
+6c
+I
+eb
+I
+71
+I
+f9
+I
+76
+I
+107
+x
+55
+/Users/murphy/ruby/coderay-0.9/test/functional/basic.rb
+p
+0
+x
+13
+attach_method
+p
+7
+I
+0
+I
+1
+I
+9
+I
+2
+I
+12
+I
+4
+I
+35
+x
+55
+/Users/murphy/ruby/coderay-0.9/test/functional/basic.rb
+p
+0
--- /dev/null
+require 'test/unit'
+$:.unshift 'lib'
+require 'coderay'
+
+begin
+ require 'rubygems' unless defined? Gem
+ gem 'RedCloth', '>= 4.0.3' rescue nil
+ require 'redcloth'
+rescue LoadError
+ warn 'RedCloth not found - skipping for_redcloth tests.'
+end
+
+class BasicTest < Test::Unit::TestCase
+
+ def test_for_redcloth
+ require 'coderay/for_redcloth'
+ assert_equal "<p><span lang=\"ruby\" class=\"CodeRay\">puts <span style=\"background-color:#fff0f0;color:#D20\"><span style=\"color:#710\">"</span><span style=\"\">Hello, World!</span><span style=\"color:#710\">"</span></span></span></p>",
+ RedCloth.new('@[ruby]puts "Hello, World!"@').to_html
+ assert_equal <<-BLOCKCODE.chomp,
+<div lang="ruby" class="CodeRay">
+ <div class="code"><pre>puts <span style="background-color:#fff0f0;color:#D20"><span style="color:#710">"</span><span style="">Hello, World!</span><span style="color:#710">"</span></span></pre></div>
+</div>
+ BLOCKCODE
+ RedCloth.new('bc[ruby]. puts "Hello, World!"').to_html
+ end
+
+ def test_for_redcloth_no_lang
+ require 'coderay/for_redcloth'
+ assert_equal "<p><code>puts \"Hello, World!\"</code></p>",
+ RedCloth.new('@puts "Hello, World!"@').to_html
+ assert_equal <<-BLOCKCODE.chomp,
+<pre><code>puts \"Hello, World!\"</code></pre>
+ BLOCKCODE
+ RedCloth.new('bc. puts "Hello, World!"').to_html
+ end
+
+ def test_for_redcloth_style
+ require 'coderay/for_redcloth'
+ assert_equal <<-BLOCKCODE.chomp,
+<pre style=\"color: red;\"><code style=\"color: red;\">puts \"Hello, World!\"</code></pre>
+ BLOCKCODE
+ RedCloth.new('bc{color: red}. puts "Hello, World!"').to_html
+ end
+
+ def test_for_redcloth_escapes
+ require 'coderay/for_redcloth'
+ assert_equal '<p><span lang="ruby" class="CodeRay">></span></p>',
+ RedCloth.new('@[ruby]>@').to_html
+ assert_equal <<-BLOCKCODE.chomp,
+<div lang="ruby" class="CodeRay">
+ <div class="code"><pre>&</pre></div>
+</div>
+ BLOCKCODE
+ RedCloth.new('bc[ruby]. &').to_html
+ end
+
+ def test_for_redcloth_escapes2
+ require 'coderay/for_redcloth'
+ assert_equal "<p><span lang=\"c\" class=\"CodeRay\"><span style=\"color:#579\">#include</span> <span style=\"color:#B44;font-weight:bold\"><test.h></span></span></p>",
+ RedCloth.new('@[c]#include <test.h>@').to_html
+ end
+
+ # See http://jgarber.lighthouseapp.com/projects/13054/tickets/124-code-markup-does-not-allow-brackets.
+ def test_for_redcloth_false_positive
+ require 'coderay/for_redcloth'
+ assert_equal '<p><code>[project]_dff.skjd</code></p>',
+ RedCloth.new('@[project]_dff.skjd@').to_html
+ # false positive, but expected behavior / known issue
+ assert_equal "<p><span lang=\"ruby\" class=\"CodeRay\">_dff.skjd</span></p>",
+ RedCloth.new('@[ruby]_dff.skjd@').to_html
+ assert_equal <<-BLOCKCODE.chomp,
+<pre><code>[project]_dff.skjd</code></pre>
+ BLOCKCODE
+ RedCloth.new('bc. [project]_dff.skjd').to_html
+ end
+
+end if defined? RedCloth
\ No newline at end of file
--- /dev/null
+!RBIX
+0
+x
+M
+1
+n
+n
+x
+10
+__script__
+i
+249
+5
+7
+0
+64
+47
+49
+1
+1
+15
+99
+43
+2
+7
+3
+49
+4
+1
+7
+5
+64
+49
+6
+1
+15
+5
+7
+7
+64
+47
+49
+1
+1
+15
+26
+93
+0
+15
+29
+144
+0
+26
+93
+1
+15
+29
+55
+0
+7
+8
+98
+9
+1
+30
+8
+61
+25
+92
+1
+27
+8
+66
+15
+7
+10
+8
+67
+1
+9
+72
+1
+8
+80
+5
+7
+11
+64
+47
+49
+1
+1
+15
+26
+93
+2
+15
+29
+102
+0
+5
+7
+12
+64
+7
+13
+64
+47
+49
+14
+2
+30
+8
+129
+26
+93
+3
+15
+24
+13
+45
+15
+16
+12
+49
+17
+1
+10
+119
+8
+124
+15
+1
+25
+8
+129
+15
+92
+3
+27
+34
+92
+2
+27
+15
+5
+7
+18
+64
+47
+49
+1
+1
+30
+8
+178
+26
+93
+4
+15
+24
+13
+45
+19
+20
+12
+49
+17
+1
+10
+161
+8
+173
+15
+5
+7
+21
+64
+47
+49
+22
+1
+25
+8
+178
+15
+92
+4
+27
+34
+92
+0
+27
+15
+26
+93
+5
+15
+29
+197
+0
+7
+23
+98
+9
+1
+30
+8
+203
+25
+92
+5
+27
+8
+208
+15
+7
+10
+8
+209
+1
+9
+245
+99
+7
+24
+45
+25
+26
+43
+27
+43
+28
+65
+49
+29
+3
+13
+99
+12
+7
+30
+12
+7
+31
+12
+65
+12
+49
+32
+4
+15
+49
+30
+0
+8
+246
+1
+15
+2
+11
+I
+c
+I
+0
+I
+0
+I
+0
+n
+p
+33
+s
+9
+test/unit
+x
+7
+require
+x
+7
+Globals
+x
+2
+$:
+x
+2
+[]
+s
+3
+lib
+x
+2
+<<
+s
+7
+coderay
+x
+3
+Gem
+x
+16
+vm_const_defined
+s
+8
+constant
+s
+8
+rubygems
+s
+8
+RedCloth
+s
+8
+>= 4.0.3
+x
+3
+gem
+x
+13
+StandardError
+n
+x
+3
+===
+s
+8
+redcloth
+x
+9
+LoadError
+n
+s
+49
+RedCloth not found - skipping for_redcloth tests.
+x
+4
+warn
+x
+8
+RedCloth
+x
+9
+BasicTest
+x
+4
+Test
+n
+x
+4
+Unit
+x
+8
+TestCase
+x
+10
+open_class
+x
+14
+__class_init__
+M
+1
+n
+n
+x
+9
+BasicTest
+i
+86
+5
+66
+99
+7
+0
+7
+1
+65
+67
+49
+2
+0
+49
+3
+4
+15
+99
+7
+4
+7
+5
+65
+67
+49
+2
+0
+49
+3
+4
+15
+99
+7
+6
+7
+7
+65
+67
+49
+2
+0
+49
+3
+4
+15
+99
+7
+8
+7
+9
+65
+67
+49
+2
+0
+49
+3
+4
+15
+99
+7
+10
+7
+11
+65
+67
+49
+2
+0
+49
+3
+4
+15
+99
+7
+12
+7
+13
+65
+67
+49
+2
+0
+49
+3
+4
+11
+I
+5
+I
+0
+I
+0
+I
+0
+n
+p
+14
+x
+17
+test_for_redcloth
+M
+1
+n
+n
+x
+17
+test_for_redcloth
+i
+96
+5
+7
+0
+64
+47
+49
+1
+1
+15
+5
+7
+2
+64
+45
+3
+4
+13
+71
+5
+47
+9
+37
+47
+49
+6
+0
+13
+7
+7
+64
+47
+49
+8
+1
+15
+8
+43
+7
+7
+64
+49
+5
+1
+49
+9
+0
+47
+49
+10
+2
+15
+5
+7
+11
+64
+49
+12
+0
+45
+3
+13
+13
+71
+5
+47
+9
+82
+47
+49
+6
+0
+13
+7
+14
+64
+47
+49
+8
+1
+15
+8
+88
+7
+14
+64
+49
+5
+1
+49
+9
+0
+47
+49
+10
+2
+11
+I
+5
+I
+0
+I
+0
+I
+0
+n
+p
+15
+s
+20
+coderay/for_redcloth
+x
+7
+require
+s
+221
+<p><span lang="ruby" class="CodeRay">puts <span style="background-color:#fff0f0;color:#D20"><span style="color:#710">"</span><span style="">Hello, World!</span><span style="color:#710">"</span></span></span></p>
+x
+8
+RedCloth
+n
+x
+3
+new
+x
+8
+allocate
+s
+28
+@[ruby]puts "Hello, World!"@
+x
+10
+initialize
+x
+7
+to_html
+x
+12
+assert_equal
+s
+252
+<div lang="ruby" class="CodeRay">
+ <div class="code"><pre>puts <span style="background-color:#fff0f0;color:#D20"><span style="color:#710">"</span><span style="">Hello, World!</span><span style="color:#710">"</span></span></pre></div>
+</div>
+
+x
+5
+chomp
+n
+s
+30
+bc[ruby]. puts "Hello, World!"
+p
+17
+I
+0
+I
+f
+I
+0
+I
+10
+I
+9
+I
+11
+I
+d
+I
+12
+I
+2e
+I
+11
+I
+33
+I
+13
+I
+3a
+I
+18
+I
+5b
+I
+13
+I
+60
+x
+62
+/Users/murphy/ruby/coderay-0.9/test/functional/for_redcloth.rb
+p
+0
+x
+17
+method_visibility
+x
+15
+add_defn_method
+x
+25
+test_for_redcloth_no_lang
+M
+1
+n
+n
+x
+25
+test_for_redcloth_no_lang
+i
+96
+5
+7
+0
+64
+47
+49
+1
+1
+15
+5
+7
+2
+64
+45
+3
+4
+13
+71
+5
+47
+9
+37
+47
+49
+6
+0
+13
+7
+7
+64
+47
+49
+8
+1
+15
+8
+43
+7
+7
+64
+49
+5
+1
+49
+9
+0
+47
+49
+10
+2
+15
+5
+7
+11
+64
+49
+12
+0
+45
+3
+13
+13
+71
+5
+47
+9
+82
+47
+49
+6
+0
+13
+7
+14
+64
+47
+49
+8
+1
+15
+8
+88
+7
+14
+64
+49
+5
+1
+49
+9
+0
+47
+49
+10
+2
+11
+I
+5
+I
+0
+I
+0
+I
+0
+n
+p
+15
+s
+20
+coderay/for_redcloth
+x
+7
+require
+s
+40
+<p><code>puts "Hello, World!"</code></p>
+x
+8
+RedCloth
+n
+x
+3
+new
+x
+8
+allocate
+s
+22
+@puts "Hello, World!"@
+x
+10
+initialize
+x
+7
+to_html
+x
+12
+assert_equal
+s
+45
+<pre><code>puts "Hello, World!"</code></pre>
+
+x
+5
+chomp
+n
+s
+24
+bc. puts "Hello, World!"
+p
+17
+I
+0
+I
+1b
+I
+0
+I
+1c
+I
+9
+I
+1d
+I
+d
+I
+1e
+I
+2e
+I
+1d
+I
+33
+I
+1f
+I
+3a
+I
+22
+I
+5b
+I
+1f
+I
+60
+x
+62
+/Users/murphy/ruby/coderay-0.9/test/functional/for_redcloth.rb
+p
+0
+x
+23
+test_for_redcloth_style
+M
+1
+n
+n
+x
+23
+test_for_redcloth_style
+i
+54
+5
+7
+0
+64
+47
+49
+1
+1
+15
+5
+7
+2
+64
+49
+3
+0
+45
+4
+5
+13
+71
+6
+47
+9
+40
+47
+49
+7
+0
+13
+7
+8
+64
+47
+49
+9
+1
+15
+8
+46
+7
+8
+64
+49
+6
+1
+49
+10
+0
+47
+49
+11
+2
+11
+I
+5
+I
+0
+I
+0
+I
+0
+n
+p
+12
+s
+20
+coderay/for_redcloth
+x
+7
+require
+s
+85
+<pre style="color: red;"><code style="color: red;">puts "Hello, World!"</code></pre>
+
+x
+5
+chomp
+x
+8
+RedCloth
+n
+x
+3
+new
+x
+8
+allocate
+s
+36
+bc{color: red}. puts "Hello, World!"
+x
+10
+initialize
+x
+7
+to_html
+x
+12
+assert_equal
+p
+11
+I
+0
+I
+25
+I
+0
+I
+26
+I
+9
+I
+27
+I
+10
+I
+2a
+I
+31
+I
+27
+I
+36
+x
+62
+/Users/murphy/ruby/coderay-0.9/test/functional/for_redcloth.rb
+p
+0
+x
+25
+test_for_redcloth_escapes
+M
+1
+n
+n
+x
+25
+test_for_redcloth_escapes
+i
+96
+5
+7
+0
+64
+47
+49
+1
+1
+15
+5
+7
+2
+64
+45
+3
+4
+13
+71
+5
+47
+9
+37
+47
+49
+6
+0
+13
+7
+7
+64
+47
+49
+8
+1
+15
+8
+43
+7
+7
+64
+49
+5
+1
+49
+9
+0
+47
+49
+10
+2
+15
+5
+7
+11
+64
+49
+12
+0
+45
+3
+13
+13
+71
+5
+47
+9
+82
+47
+49
+6
+0
+13
+7
+14
+64
+47
+49
+8
+1
+15
+8
+88
+7
+14
+64
+49
+5
+1
+49
+9
+0
+47
+49
+10
+2
+11
+I
+5
+I
+0
+I
+0
+I
+0
+n
+p
+15
+s
+20
+coderay/for_redcloth
+x
+7
+require
+s
+52
+<p><span lang="ruby" class="CodeRay">></span></p>
+x
+8
+RedCloth
+n
+x
+3
+new
+x
+8
+allocate
+s
+9
+@[ruby]>@
+x
+10
+initialize
+x
+7
+to_html
+x
+12
+assert_equal
+s
+84
+<div lang="ruby" class="CodeRay">
+ <div class="code"><pre>&</pre></div>
+</div>
+
+x
+5
+chomp
+n
+s
+11
+bc[ruby]. &
+p
+17
+I
+0
+I
+2d
+I
+0
+I
+2e
+I
+9
+I
+2f
+I
+d
+I
+30
+I
+2e
+I
+2f
+I
+33
+I
+31
+I
+3a
+I
+36
+I
+5b
+I
+31
+I
+60
+x
+62
+/Users/murphy/ruby/coderay-0.9/test/functional/for_redcloth.rb
+p
+0
+x
+26
+test_for_redcloth_escapes2
+M
+1
+n
+n
+x
+26
+test_for_redcloth_escapes2
+i
+51
+5
+7
+0
+64
+47
+49
+1
+1
+15
+5
+7
+2
+64
+45
+3
+4
+13
+71
+5
+47
+9
+37
+47
+49
+6
+0
+13
+7
+7
+64
+47
+49
+8
+1
+15
+8
+43
+7
+7
+64
+49
+5
+1
+49
+9
+0
+47
+49
+10
+2
+11
+I
+5
+I
+0
+I
+0
+I
+0
+n
+p
+11
+s
+20
+coderay/for_redcloth
+x
+7
+require
+s
+149
+<p><span lang="c" class="CodeRay"><span style="color:#579">#include</span> <span style="color:#B44;font-weight:bold"><test.h></span></span></p>
+x
+8
+RedCloth
+n
+x
+3
+new
+x
+8
+allocate
+s
+22
+@[c]#include <test.h>@
+x
+10
+initialize
+x
+7
+to_html
+x
+12
+assert_equal
+p
+11
+I
+0
+I
+39
+I
+0
+I
+3a
+I
+9
+I
+3b
+I
+d
+I
+3c
+I
+2e
+I
+3b
+I
+33
+x
+62
+/Users/murphy/ruby/coderay-0.9/test/functional/for_redcloth.rb
+p
+0
+x
+32
+test_for_redcloth_false_positive
+M
+1
+n
+n
+x
+32
+test_for_redcloth_false_positive
+i
+138
+5
+7
+0
+64
+47
+49
+1
+1
+15
+5
+7
+2
+64
+45
+3
+4
+13
+71
+5
+47
+9
+37
+47
+49
+6
+0
+13
+7
+7
+64
+47
+49
+8
+1
+15
+8
+43
+7
+7
+64
+49
+5
+1
+49
+9
+0
+47
+49
+10
+2
+15
+5
+7
+11
+64
+45
+3
+12
+13
+71
+5
+47
+9
+79
+47
+49
+6
+0
+13
+7
+13
+64
+47
+49
+8
+1
+15
+8
+85
+7
+13
+64
+49
+5
+1
+49
+9
+0
+47
+49
+10
+2
+15
+5
+7
+14
+64
+49
+15
+0
+45
+3
+16
+13
+71
+5
+47
+9
+124
+47
+49
+6
+0
+13
+7
+17
+64
+47
+49
+8
+1
+15
+8
+130
+7
+17
+64
+49
+5
+1
+49
+9
+0
+47
+49
+10
+2
+11
+I
+5
+I
+0
+I
+0
+I
+0
+n
+p
+18
+s
+20
+coderay/for_redcloth
+x
+7
+require
+s
+38
+<p><code>[project]_dff.skjd</code></p>
+x
+8
+RedCloth
+n
+x
+3
+new
+x
+8
+allocate
+s
+20
+@[project]_dff.skjd@
+x
+10
+initialize
+x
+7
+to_html
+x
+12
+assert_equal
+s
+57
+<p><span lang="ruby" class="CodeRay">_dff.skjd</span></p>
+n
+s
+17
+@[ruby]_dff.skjd@
+s
+43
+<pre><code>[project]_dff.skjd</code></pre>
+
+x
+5
+chomp
+n
+s
+22
+bc. [project]_dff.skjd
+p
+23
+I
+0
+I
+40
+I
+0
+I
+41
+I
+9
+I
+42
+I
+d
+I
+43
+I
+2e
+I
+42
+I
+33
+I
+45
+I
+37
+I
+46
+I
+58
+I
+45
+I
+5d
+I
+47
+I
+64
+I
+4a
+I
+85
+I
+47
+I
+8a
+x
+62
+/Users/murphy/ruby/coderay-0.9/test/functional/for_redcloth.rb
+p
+0
+p
+13
+I
+2
+I
+f
+I
+10
+I
+1b
+I
+1e
+I
+25
+I
+2c
+I
+2d
+I
+3a
+I
+39
+I
+48
+I
+40
+I
+56
+x
+62
+/Users/murphy/ruby/coderay-0.9/test/functional/for_redcloth.rb
+p
+0
+x
+13
+attach_method
+p
+23
+I
+0
+I
+1
+I
+9
+I
+2
+I
+18
+I
+3
+I
+21
+I
+6
+I
+51
+I
+7
+I
+85
+I
+8
+I
+95
+I
+9
+I
+a2
+I
+a
+I
+b6
+I
+4d
+I
+d3
+I
+d
+I
+f5
+I
+4d
+I
+f9
+x
+62
+/Users/murphy/ruby/coderay-0.9/test/functional/for_redcloth.rb
+p
+0
--- /dev/null
+require 'test/unit'
+require 'coderay'
+
+class PluginScannerTest < Test::Unit::TestCase
+
+ def test_load
+ require File.join(File.dirname(__FILE__), 'vhdl')
+ assert_equal 'VHDL', CodeRay.scanner(:vhdl).class.name
+ end
+
+end
--- /dev/null
+!RBIX
+0
+x
+M
+1
+n
+n
+x
+10
+__script__
+i
+53
+5
+7
+0
+64
+47
+49
+1
+1
+15
+5
+7
+2
+64
+47
+49
+1
+1
+15
+99
+7
+3
+45
+4
+5
+43
+6
+43
+7
+65
+49
+8
+3
+13
+99
+12
+7
+9
+12
+7
+10
+12
+65
+12
+49
+11
+4
+15
+49
+9
+0
+15
+2
+11
+I
+6
+I
+0
+I
+0
+I
+0
+n
+p
+12
+s
+9
+test/unit
+x
+7
+require
+s
+7
+coderay
+x
+17
+PluginScannerTest
+x
+4
+Test
+n
+x
+4
+Unit
+x
+8
+TestCase
+x
+10
+open_class
+x
+14
+__class_init__
+M
+1
+n
+n
+x
+17
+PluginScannerTest
+i
+16
+5
+66
+99
+7
+0
+7
+1
+65
+67
+49
+2
+0
+49
+3
+4
+11
+I
+5
+I
+0
+I
+0
+I
+0
+n
+p
+4
+x
+9
+test_load
+M
+1
+n
+n
+x
+9
+test_load
+i
+48
+5
+45
+0
+1
+45
+0
+2
+65
+49
+3
+0
+49
+4
+1
+7
+5
+64
+49
+6
+2
+47
+49
+7
+1
+15
+5
+7
+8
+64
+45
+9
+10
+7
+11
+49
+12
+1
+49
+13
+0
+49
+14
+0
+47
+49
+15
+2
+11
+I
+4
+I
+0
+I
+0
+I
+0
+n
+p
+16
+x
+4
+File
+n
+n
+x
+11
+active_path
+x
+7
+dirname
+s
+4
+vhdl
+x
+4
+join
+x
+7
+require
+s
+4
+VHDL
+x
+7
+CodeRay
+n
+x
+4
+vhdl
+x
+7
+scanner
+x
+5
+class
+x
+4
+name
+x
+12
+assert_equal
+p
+7
+I
+0
+I
+6
+I
+0
+I
+7
+I
+19
+I
+8
+I
+30
+x
+69
+/Users/murphy/ruby/coderay-0.9/test/functional/load_plugin_scanner.rb
+p
+0
+x
+17
+method_visibility
+x
+15
+add_defn_method
+p
+3
+I
+2
+I
+6
+I
+10
+x
+69
+/Users/murphy/ruby/coderay-0.9/test/functional/load_plugin_scanner.rb
+p
+0
+x
+13
+attach_method
+p
+7
+I
+0
+I
+1
+I
+9
+I
+2
+I
+12
+I
+4
+I
+35
+x
+69
+/Users/murphy/ruby/coderay-0.9/test/functional/load_plugin_scanner.rb
+p
+0
--- /dev/null
+require 'test/unit'
+
+MYDIR = File.dirname(__FILE__)
+
+$:.unshift 'lib'
+require 'coderay'
+puts "Running basic CodeRay #{CodeRay::VERSION} tests..."
+
+suite = %w(basic load_plugin_scanner word_list)
+for test_case in suite
+ load File.join(MYDIR, test_case + '.rb')
+end
--- /dev/null
+!RBIX
+0
+x
+M
+1
+n
+n
+x
+10
+__script__
+i
+95
+5
+7
+0
+64
+47
+49
+1
+1
+15
+65
+7
+2
+45
+3
+4
+65
+49
+5
+0
+49
+6
+1
+49
+7
+2
+15
+99
+43
+8
+7
+9
+49
+10
+1
+7
+11
+64
+49
+12
+1
+15
+5
+7
+13
+64
+47
+49
+1
+1
+15
+5
+7
+14
+45
+15
+16
+43
+17
+47
+49
+18
+0
+7
+19
+63
+3
+47
+49
+20
+1
+15
+7
+21
+64
+7
+22
+64
+7
+23
+64
+35
+3
+19
+0
+15
+20
+0
+56
+24
+50
+25
+0
+15
+2
+11
+I
+6
+I
+2
+I
+0
+I
+0
+n
+p
+26
+s
+9
+test/unit
+x
+7
+require
+x
+5
+MYDIR
+x
+4
+File
+n
+x
+11
+active_path
+x
+7
+dirname
+x
+9
+const_set
+x
+7
+Globals
+x
+2
+$:
+x
+2
+[]
+s
+3
+lib
+x
+2
+<<
+s
+7
+coderay
+s
+22
+Running basic CodeRay
+x
+7
+CodeRay
+n
+x
+7
+VERSION
+x
+4
+to_s
+s
+9
+ tests...
+x
+4
+puts
+s
+5
+basic
+s
+19
+load_plugin_scanner
+s
+9
+word_list
+M
+1
+p
+2
+x
+9
+for_block
+t
+n
+x
+9
+__block__
+i
+28
+57
+22
+1
+1
+15
+5
+45
+0
+1
+45
+2
+3
+21
+1
+1
+7
+4
+64
+81
+5
+49
+6
+2
+47
+49
+7
+1
+11
+I
+6
+I
+0
+I
+1
+I
+1
+n
+p
+8
+x
+4
+File
+n
+x
+5
+MYDIR
+n
+s
+3
+.rb
+x
+1
++
+x
+4
+join
+x
+4
+load
+p
+5
+I
+0
+I
+a
+I
+5
+I
+b
+I
+1c
+x
+55
+/Users/murphy/ruby/coderay-0.9/test/functional/suite.rb
+p
+0
+x
+4
+each
+p
+15
+I
+0
+I
+1
+I
+9
+I
+3
+I
+1a
+I
+5
+I
+29
+I
+6
+I
+32
+I
+7
+I
+47
+I
+9
+I
+55
+I
+a
+I
+5f
+x
+55
+/Users/murphy/ruby/coderay-0.9/test/functional/suite.rb
+p
+2
+x
+5
+suite
+x
+9
+test_case
--- /dev/null
+class VHDL < CodeRay::Scanners::Scanner
+
+ register_for :vhdl
+
+ RESERVED_WORDS = [
+ 'access','after','alias','all','assert','architecture','begin',
+ 'block','body','buffer','bus','case','component','configuration','constant',
+ 'disconnect','downto','else','elsif','end','entity','exit','file','for',
+ 'function','generate','generic','group','guarded','if','impure','in',
+ 'inertial','inout','is','label','library','linkage','literal','loop',
+ 'map','new','next','null','of','on','open','others','out','package',
+ 'port','postponed','procedure','process','pure','range','record','register',
+ 'reject','report','return','select','severity','signal','shared','subtype',
+ 'then','to','transport','type','unaffected','units','until','use','variable',
+ 'wait','when','while','with','note','warning','error','failure','and',
+ 'or','xor','not','nor',
+ 'array'
+ ]
+
+ PREDEFINED_TYPES = [
+ 'bit','bit_vector','character','boolean','integer','real','time','string',
+ 'severity_level','positive','natural','signed','unsigned','line','text',
+ 'std_logic','std_logic_vector','std_ulogic','std_ulogic_vector','qsim_state',
+ 'qsim_state_vector','qsim_12state','qsim_12state_vector','qsim_strength',
+ 'mux_bit','mux_vector','reg_bit','reg_vector','wor_bit','wor_vector'
+ ]
+
+ PREDEFINED_CONSTANTS = [
+
+ ]
+
+ IDENT_KIND = CodeRay::CaseIgnoringWordList.new(:ident).
+ add(RESERVED_WORDS, :reserved).
+ add(PREDEFINED_TYPES, :pre_type).
+ add(PREDEFINED_CONSTANTS, :pre_constant)
+
+ ESCAPE = / [rbfntv\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} /x
+ UNICODE_ESCAPE = / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} /x
+
+ def scan_tokens tokens, options
+
+ state = :initial
+
+ until eos?
+
+ kind = nil
+ match = nil
+
+ case state
+
+ when :initial
+
+ if scan(/ \s+ | \\\n /x)
+ kind = :space
+
+ elsif scan(/-- .*/x)
+ kind = :comment
+
+ elsif scan(/ [-+*\/=<>?:;,!&^|()\[\]{}~%]+ | \.(?!\d) /x)
+ kind = :operator
+
+ elsif match = scan(/ [A-Za-z_][A-Za-z_0-9]* /x)
+ kind = IDENT_KIND[match.downcase]
+
+ elsif match = scan(/[a-z]?"/i)
+ tokens << [:open, :string]
+ state = :string
+ kind = :delimiter
+
+ elsif scan(/ L?' (?: [^\'\n\\] | \\ #{ESCAPE} )? '? /ox)
+ kind = :char
+
+ elsif scan(/(?:\d+)(?![.eEfF])/)
+ kind = :integer
+
+ elsif scan(/\d[fF]?|\d*\.\d+(?:[eE][+-]?\d+)?[fF]?|\d+[eE][+-]?\d+[fF]?/)
+ kind = :float
+
+ else
+ getch
+ kind = :error
+
+ end
+
+ when :string
+ if scan(/[^\\\n"]+/)
+ kind = :content
+ elsif scan(/"/)
+ tokens << ['"', :delimiter]
+ tokens << [:close, :string]
+ state = :initial
+ next
+ elsif scan(/ \\ (?: #{ESCAPE} | #{UNICODE_ESCAPE} ) /mox)
+ kind = :char
+ elsif scan(/ \\ | $ /x)
+ tokens << [:close, :string]
+ kind = :error
+ state = :initial
+ else
+ raise_inspect "else case \" reached; %p not handled." % peek(1), tokens
+ end
+
+ else
+ raise_inspect 'Unknown state', tokens
+
+ end
+
+ match ||= matched
+ if $DEBUG and not kind
+ raise_inspect 'Error token %p in line %d' %
+ [[match, kind], line], tokens
+ end
+ raise_inspect 'Empty token', tokens unless match
+
+ tokens << [match, kind]
+
+ end
+
+ if state == :string
+ tokens << [:close, :string]
+ end
+
+ tokens
+ end
+
+end
--- /dev/null
+!RBIX
+0
+x
+M
+1
+n
+n
+x
+10
+__script__
+i
+35
+99
+7
+0
+45
+1
+2
+43
+3
+43
+4
+65
+49
+5
+3
+13
+99
+12
+7
+6
+12
+7
+7
+12
+65
+12
+49
+8
+4
+15
+49
+6
+0
+15
+2
+11
+I
+6
+I
+0
+I
+0
+I
+0
+n
+p
+9
+x
+4
+VHDL
+x
+7
+CodeRay
+n
+x
+8
+Scanners
+x
+7
+Scanner
+x
+10
+open_class
+x
+14
+__class_init__
+M
+1
+n
+n
+x
+4
+VHDL
+i
+519
+5
+66
+5
+7
+0
+47
+49
+1
+1
+15
+65
+7
+2
+7
+3
+64
+7
+4
+64
+7
+5
+64
+7
+6
+64
+7
+7
+64
+7
+8
+64
+7
+9
+64
+7
+10
+64
+7
+11
+64
+7
+12
+64
+7
+13
+64
+7
+14
+64
+7
+15
+64
+7
+16
+64
+7
+17
+64
+7
+18
+64
+7
+19
+64
+7
+20
+64
+7
+21
+64
+7
+22
+64
+7
+23
+64
+7
+24
+64
+7
+25
+64
+7
+26
+64
+7
+27
+64
+7
+28
+64
+7
+29
+64
+7
+30
+64
+7
+31
+64
+7
+32
+64
+7
+33
+64
+7
+34
+64
+7
+35
+64
+7
+36
+64
+7
+37
+64
+7
+38
+64
+7
+39
+64
+7
+40
+64
+7
+41
+64
+7
+42
+64
+7
+43
+64
+7
+44
+64
+7
+45
+64
+7
+46
+64
+7
+47
+64
+7
+48
+64
+7
+49
+64
+7
+50
+64
+7
+51
+64
+7
+52
+64
+7
+53
+64
+7
+54
+64
+7
+55
+64
+7
+56
+64
+7
+57
+64
+7
+58
+64
+7
+59
+64
+7
+60
+64
+7
+61
+64
+7
+62
+64
+7
+63
+64
+7
+64
+64
+7
+65
+64
+7
+66
+64
+7
+67
+64
+7
+68
+64
+7
+69
+64
+7
+70
+64
+7
+71
+64
+7
+72
+64
+7
+73
+64
+7
+74
+64
+7
+75
+64
+7
+76
+64
+7
+77
+64
+7
+78
+64
+7
+79
+64
+7
+80
+64
+7
+81
+64
+7
+82
+64
+7
+83
+64
+7
+84
+64
+7
+85
+64
+7
+86
+64
+7
+87
+64
+7
+88
+64
+7
+89
+64
+7
+90
+64
+7
+91
+64
+35
+89
+49
+92
+2
+15
+65
+7
+93
+7
+94
+64
+7
+95
+64
+7
+96
+64
+7
+97
+64
+7
+98
+64
+7
+99
+64
+7
+100
+64
+7
+101
+64
+7
+102
+64
+7
+103
+64
+7
+104
+64
+7
+105
+64
+7
+106
+64
+7
+107
+64
+7
+108
+64
+7
+109
+64
+7
+110
+64
+7
+111
+64
+7
+112
+64
+7
+113
+64
+7
+114
+64
+7
+115
+64
+7
+116
+64
+7
+117
+64
+7
+118
+64
+7
+119
+64
+7
+120
+64
+7
+121
+64
+7
+122
+64
+7
+123
+64
+35
+30
+49
+92
+2
+15
+65
+7
+124
+35
+0
+49
+92
+2
+15
+65
+7
+125
+45
+126
+127
+43
+128
+13
+71
+129
+47
+9
+422
+47
+49
+130
+0
+13
+7
+131
+47
+49
+132
+1
+15
+8
+427
+7
+131
+49
+129
+1
+45
+2
+133
+7
+134
+49
+135
+2
+45
+93
+136
+7
+137
+49
+135
+2
+45
+124
+138
+7
+139
+49
+135
+2
+49
+92
+2
+15
+65
+7
+140
+7
+141
+13
+70
+9
+476
+15
+44
+43
+142
+7
+143
+80
+49
+129
+2
+6
+141
+49
+92
+2
+15
+65
+7
+144
+7
+145
+13
+70
+9
+501
+15
+44
+43
+142
+7
+146
+80
+49
+129
+2
+6
+145
+49
+92
+2
+15
+99
+7
+147
+7
+148
+65
+67
+49
+149
+0
+49
+150
+4
+11
+I
+5b
+I
+0
+I
+0
+I
+0
+n
+p
+151
+x
+4
+vhdl
+x
+12
+register_for
+x
+14
+RESERVED_WORDS
+s
+6
+access
+s
+5
+after
+s
+5
+alias
+s
+3
+all
+s
+6
+assert
+s
+12
+architecture
+s
+5
+begin
+s
+5
+block
+s
+4
+body
+s
+6
+buffer
+s
+3
+bus
+s
+4
+case
+s
+9
+component
+s
+13
+configuration
+s
+8
+constant
+s
+10
+disconnect
+s
+6
+downto
+s
+4
+else
+s
+5
+elsif
+s
+3
+end
+s
+6
+entity
+s
+4
+exit
+s
+4
+file
+s
+3
+for
+s
+8
+function
+s
+8
+generate
+s
+7
+generic
+s
+5
+group
+s
+7
+guarded
+s
+2
+if
+s
+6
+impure
+s
+2
+in
+s
+8
+inertial
+s
+5
+inout
+s
+2
+is
+s
+5
+label
+s
+7
+library
+s
+7
+linkage
+s
+7
+literal
+s
+4
+loop
+s
+3
+map
+s
+3
+new
+s
+4
+next
+s
+4
+null
+s
+2
+of
+s
+2
+on
+s
+4
+open
+s
+6
+others
+s
+3
+out
+s
+7
+package
+s
+4
+port
+s
+9
+postponed
+s
+9
+procedure
+s
+7
+process
+s
+4
+pure
+s
+5
+range
+s
+6
+record
+s
+8
+register
+s
+6
+reject
+s
+6
+report
+s
+6
+return
+s
+6
+select
+s
+8
+severity
+s
+6
+signal
+s
+6
+shared
+s
+7
+subtype
+s
+4
+then
+s
+2
+to
+s
+9
+transport
+s
+4
+type
+s
+10
+unaffected
+s
+5
+units
+s
+5
+until
+s
+3
+use
+s
+8
+variable
+s
+4
+wait
+s
+4
+when
+s
+5
+while
+s
+4
+with
+s
+4
+note
+s
+7
+warning
+s
+5
+error
+s
+7
+failure
+s
+3
+and
+s
+2
+or
+s
+3
+xor
+s
+3
+not
+s
+3
+nor
+s
+5
+array
+x
+9
+const_set
+x
+16
+PREDEFINED_TYPES
+s
+3
+bit
+s
+10
+bit_vector
+s
+9
+character
+s
+7
+boolean
+s
+7
+integer
+s
+4
+real
+s
+4
+time
+s
+6
+string
+s
+14
+severity_level
+s
+8
+positive
+s
+7
+natural
+s
+6
+signed
+s
+8
+unsigned
+s
+4
+line
+s
+4
+text
+s
+9
+std_logic
+s
+16
+std_logic_vector
+s
+10
+std_ulogic
+s
+17
+std_ulogic_vector
+s
+10
+qsim_state
+s
+17
+qsim_state_vector
+s
+12
+qsim_12state
+s
+19
+qsim_12state_vector
+s
+13
+qsim_strength
+s
+7
+mux_bit
+s
+10
+mux_vector
+s
+7
+reg_bit
+s
+10
+reg_vector
+s
+7
+wor_bit
+s
+10
+wor_vector
+x
+20
+PREDEFINED_CONSTANTS
+x
+10
+IDENT_KIND
+x
+7
+CodeRay
+n
+x
+20
+CaseIgnoringWordList
+x
+3
+new
+x
+8
+allocate
+x
+5
+ident
+x
+10
+initialize
+n
+x
+8
+reserved
+x
+3
+add
+n
+x
+8
+pre_type
+n
+x
+12
+pre_constant
+x
+6
+ESCAPE
+n
+x
+6
+Regexp
+s
+49
+ [rbfntv\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3}
+x
+14
+UNICODE_ESCAPE
+n
+s
+35
+ u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8}
+x
+11
+scan_tokens
+M
+1
+n
+n
+x
+11
+scan_tokens
+i
+688
+7
+0
+19
+2
+15
+5
+47
+49
+1
+0
+10
+660
+1
+19
+3
+15
+1
+19
+4
+15
+20
+2
+13
+7
+0
+12
+49
+2
+1
+9
+331
+15
+5
+7
+3
+13
+70
+9
+51
+15
+44
+43
+4
+7
+5
+80
+49
+6
+2
+6
+3
+47
+49
+7
+1
+9
+63
+7
+8
+19
+3
+8
+329
+5
+7
+9
+13
+70
+9
+82
+15
+44
+43
+4
+7
+10
+80
+49
+6
+2
+6
+9
+47
+49
+7
+1
+9
+94
+7
+11
+19
+3
+8
+329
+5
+7
+12
+13
+70
+9
+113
+15
+44
+43
+4
+7
+13
+80
+49
+6
+2
+6
+12
+47
+49
+7
+1
+9
+125
+7
+14
+19
+3
+8
+329
+5
+7
+15
+13
+70
+9
+144
+15
+44
+43
+4
+7
+16
+80
+49
+6
+2
+6
+15
+47
+49
+7
+1
+19
+4
+9
+167
+45
+17
+18
+20
+4
+49
+19
+0
+49
+20
+1
+19
+3
+8
+329
+5
+7
+21
+13
+70
+9
+186
+15
+44
+43
+4
+7
+22
+79
+49
+6
+2
+6
+21
+47
+49
+7
+1
+19
+4
+9
+217
+20
+0
+7
+23
+7
+24
+35
+2
+49
+25
+1
+15
+7
+24
+19
+2
+15
+7
+26
+19
+3
+8
+329
+5
+7
+27
+13
+70
+9
+247
+15
+44
+43
+4
+7
+28
+45
+29
+30
+47
+49
+31
+0
+7
+32
+63
+3
+80
+49
+6
+2
+6
+27
+47
+49
+7
+1
+9
+259
+7
+33
+19
+3
+8
+329
+5
+7
+34
+13
+70
+9
+278
+15
+44
+43
+4
+7
+35
+78
+49
+6
+2
+6
+34
+47
+49
+7
+1
+9
+290
+7
+36
+19
+3
+8
+329
+5
+7
+37
+13
+70
+9
+309
+15
+44
+43
+4
+7
+38
+78
+49
+6
+2
+6
+37
+47
+49
+7
+1
+9
+321
+7
+39
+19
+3
+8
+329
+5
+48
+40
+15
+7
+41
+19
+3
+8
+564
+13
+7
+24
+12
+49
+2
+1
+9
+553
+15
+5
+7
+42
+13
+70
+9
+360
+15
+44
+43
+4
+7
+43
+78
+49
+6
+2
+6
+42
+47
+49
+7
+1
+9
+372
+7
+44
+19
+3
+8
+551
+5
+7
+45
+13
+70
+9
+391
+15
+44
+43
+4
+7
+46
+78
+49
+6
+2
+6
+45
+47
+49
+7
+1
+9
+432
+20
+0
+7
+46
+64
+7
+26
+35
+2
+49
+25
+1
+15
+20
+0
+7
+47
+7
+24
+35
+2
+49
+25
+1
+15
+7
+0
+19
+2
+15
+1
+8
+656
+8
+551
+5
+7
+48
+13
+70
+9
+472
+15
+44
+43
+4
+7
+49
+45
+29
+50
+47
+49
+31
+0
+7
+51
+45
+52
+53
+47
+49
+31
+0
+7
+54
+63
+5
+4
+6
+49
+6
+2
+6
+48
+47
+49
+7
+1
+9
+484
+7
+33
+19
+3
+8
+551
+5
+7
+55
+13
+70
+9
+503
+15
+44
+43
+4
+7
+56
+80
+49
+6
+2
+6
+55
+47
+49
+7
+1
+9
+532
+20
+0
+7
+47
+7
+24
+35
+2
+49
+25
+1
+15
+7
+41
+19
+3
+15
+7
+0
+19
+2
+8
+551
+5
+7
+57
+64
+5
+79
+47
+49
+58
+1
+49
+59
+1
+20
+0
+47
+49
+60
+2
+8
+564
+15
+5
+7
+61
+64
+20
+0
+47
+49
+60
+2
+15
+20
+4
+13
+10
+576
+15
+5
+48
+62
+19
+4
+15
+99
+43
+63
+7
+64
+49
+20
+1
+13
+9
+597
+15
+20
+3
+10
+596
+2
+8
+597
+3
+9
+625
+5
+7
+65
+64
+20
+4
+20
+3
+35
+2
+5
+48
+66
+35
+2
+49
+59
+1
+20
+0
+47
+49
+60
+2
+8
+626
+1
+15
+20
+4
+9
+634
+1
+8
+644
+5
+7
+67
+64
+20
+0
+47
+49
+60
+2
+15
+20
+0
+20
+4
+20
+3
+35
+2
+49
+25
+1
+15
+68
+8
+5
+1
+15
+20
+2
+7
+24
+83
+68
+9
+683
+20
+0
+7
+47
+7
+24
+35
+2
+49
+25
+1
+8
+684
+1
+15
+20
+0
+11
+I
+c
+I
+5
+I
+2
+I
+2
+n
+p
+69
+x
+7
+initial
+x
+4
+eos?
+x
+3
+===
+n
+x
+6
+Regexp
+s
+12
+ \s+ | \\\n
+x
+3
+new
+x
+4
+scan
+x
+5
+space
+n
+s
+5
+-- .*
+x
+7
+comment
+n
+s
+42
+ [-+*\/=<>?:;,!&^|()\[\]{}~%]+ | \.(?!\d)
+x
+8
+operator
+n
+s
+24
+ [A-Za-z_][A-Za-z_0-9]*
+x
+10
+IDENT_KIND
+n
+x
+8
+downcase
+x
+2
+[]
+n
+s
+7
+[a-z]?"
+x
+4
+open
+x
+6
+string
+x
+2
+<<
+x
+9
+delimiter
+n
+s
+24
+ L?' (?: [^\'\n\\] | \\
+x
+6
+ESCAPE
+n
+x
+4
+to_s
+s
+7
+ )? '?
+x
+4
+char
+n
+s
+18
+(?:\d+)(?![.eEfF])
+x
+7
+integer
+n
+s
+59
+\d[fF]?|\d*\.\d+(?:[eE][+-]?\d+)?[fF]?|\d+[eE][+-]?\d+[fF]?
+x
+5
+float
+x
+5
+getch
+x
+5
+error
+n
+s
+9
+[^\\\n"]+
+x
+7
+content
+n
+s
+1
+"
+x
+5
+close
+n
+s
+8
+ \\ (?:
+n
+s
+3
+ |
+x
+14
+UNICODE_ESCAPE
+n
+s
+3
+ )
+n
+s
+8
+ \\ | $
+s
+36
+else case " reached; %p not handled.
+x
+4
+peek
+x
+1
+%
+x
+13
+raise_inspect
+s
+13
+Unknown state
+x
+7
+matched
+x
+7
+Globals
+x
+6
+$DEBUG
+s
+25
+Error token %p in line %d
+x
+4
+line
+s
+11
+Empty token
+x
+2
+==
+p
+111
+I
+0
+I
+28
+I
+0
+I
+2a
+I
+5
+I
+2c
+I
+c
+I
+2e
+I
+10
+I
+2f
+I
+14
+I
+31
+I
+17
+I
+33
+I
+20
+I
+35
+I
+39
+I
+36
+I
+3f
+I
+38
+I
+58
+I
+39
+I
+5e
+I
+3b
+I
+77
+I
+3c
+I
+7d
+I
+3e
+I
+98
+I
+3f
+I
+a7
+I
+41
+I
+c2
+I
+42
+I
+ce
+I
+43
+I
+d3
+I
+44
+I
+d9
+I
+46
+I
+fd
+I
+47
+I
+103
+I
+49
+I
+11c
+I
+4a
+I
+122
+I
+4c
+I
+13b
+I
+4d
+I
+141
+I
+50
+I
+145
+I
+51
+I
+14c
+I
+55
+I
+155
+I
+56
+I
+16e
+I
+57
+I
+174
+I
+58
+I
+18d
+I
+59
+I
+19a
+I
+5a
+I
+1a6
+I
+5b
+I
+1ab
+I
+5c
+I
+1b0
+I
+5d
+I
+1de
+I
+5e
+I
+1e4
+I
+5f
+I
+1fd
+I
+60
+I
+209
+I
+61
+I
+20e
+I
+62
+I
+214
+I
+64
+I
+22a
+I
+68
+I
+235
+I
+6c
+I
+241
+I
+6d
+I
+257
+I
+6f
+I
+258
+I
+6e
+I
+25b
+I
+6f
+I
+271
+I
+6d
+I
+273
+I
+71
+I
+285
+I
+73
+I
+296
+I
+77
+I
+29e
+I
+78
+I
+2ab
+I
+77
+I
+2ad
+I
+7b
+I
+2b0
+x
+54
+/Users/murphy/ruby/coderay-0.9/test/functional/vhdl.rb
+p
+5
+x
+6
+tokens
+x
+7
+options
+x
+5
+state
+x
+4
+kind
+x
+5
+match
+x
+17
+method_visibility
+x
+15
+add_defn_method
+p
+65
+I
+2
+I
+3
+I
+a
+I
+5
+I
+d
+I
+6
+I
+22
+I
+7
+I
+3a
+I
+8
+I
+55
+I
+9
+I
+6d
+I
+a
+I
+85
+I
+b
+I
+a3
+I
+c
+I
+bb
+I
+d
+I
+d3
+I
+e
+I
+ee
+I
+f
+I
+109
+I
+10
+I
+115
+I
+11
+I
+11e
+I
+14
+I
+121
+I
+15
+I
+139
+I
+16
+I
+14e
+I
+17
+I
+15d
+I
+18
+I
+169
+I
+19
+I
+181
+I
+1c
+I
+184
+I
+1e
+I
+18a
+I
+20
+I
+1ab
+I
+21
+I
+1b0
+I
+20
+I
+1b3
+I
+22
+I
+1b8
+I
+20
+I
+1bb
+I
+23
+I
+1c0
+I
+20
+I
+1c7
+I
+25
+I
+1e0
+I
+26
+I
+1f9
+I
+28
+I
+207
+x
+54
+/Users/murphy/ruby/coderay-0.9/test/functional/vhdl.rb
+p
+0
+x
+13
+attach_method
+p
+3
+I
+0
+I
+1
+I
+23
+x
+54
+/Users/murphy/ruby/coderay-0.9/test/functional/vhdl.rb
+p
+0
--- /dev/null
+require 'test/unit'
+require 'coderay'
+
+class WordListTest < Test::Unit::TestCase
+
+ include CodeRay
+
+ # define word arrays
+ RESERVED_WORDS = %w[
+ asm break case continue default do else
+ ...
+ ]
+
+ PREDEFINED_TYPES = %w[
+ int long short char void
+ ...
+ ]
+
+ PREDEFINED_CONSTANTS = %w[
+ EOF NULL ...
+ ]
+
+ # make a WordList
+ IDENT_KIND = WordList.new(:ident).
+ add(RESERVED_WORDS, :reserved).
+ add(PREDEFINED_TYPES, :pre_type).
+ add(PREDEFINED_CONSTANTS, :pre_constant)
+
+ def test_word_list_example
+ assert_equal :pre_type, IDENT_KIND['void']
+ # assert_equal :pre_constant, IDENT_KIND['...'] # not specified
+ end
+
+ def test_word_list
+ list = WordList.new(:ident).add(['foobar'], :reserved)
+ assert_equal :reserved, list['foobar']
+ assert_equal :ident, list['FooBar']
+ end
+
+ def test_word_list_cached
+ list = WordList.new(:ident, true).add(['foobar'], :reserved)
+ assert_equal :reserved, list['foobar']
+ assert_equal :ident, list['FooBar']
+ end
+
+ def test_case_ignoring_word_list
+ list = CaseIgnoringWordList.new(:ident).add(['foobar'], :reserved)
+ assert_equal :ident, list['foo']
+ assert_equal :reserved, list['foobar']
+ assert_equal :reserved, list['FooBar']
+
+ list = CaseIgnoringWordList.new(:ident).add(['FooBar'], :reserved)
+ assert_equal :ident, list['foo']
+ assert_equal :reserved, list['foobar']
+ assert_equal :reserved, list['FooBar']
+ end
+
+ def test_case_ignoring_word_list_cached
+ list = CaseIgnoringWordList.new(:ident, true).add(['foobar'], :reserved)
+ assert_equal :ident, list['foo']
+ assert_equal :reserved, list['foobar']
+ assert_equal :reserved, list['FooBar']
+
+ list = CaseIgnoringWordList.new(:ident, true).add(['FooBar'], :reserved)
+ assert_equal :ident, list['foo']
+ assert_equal :reserved, list['foobar']
+ assert_equal :reserved, list['FooBar']
+ end
+
+ def test_dup
+ list = WordList.new(:ident).add(['foobar'], :reserved)
+ assert_equal :reserved, list['foobar']
+ list2 = list.dup
+ list2.add(%w[foobar], :keyword)
+ assert_equal :keyword, list2['foobar']
+ assert_equal :reserved, list['foobar']
+ end
+
+end
\ No newline at end of file
--- /dev/null
+!RBIX
+0
+x
+M
+1
+n
+n
+x
+10
+__script__
+i
+53
+5
+7
+0
+64
+47
+49
+1
+1
+15
+5
+7
+2
+64
+47
+49
+1
+1
+15
+99
+7
+3
+45
+4
+5
+43
+6
+43
+7
+65
+49
+8
+3
+13
+99
+12
+7
+9
+12
+7
+10
+12
+65
+12
+49
+11
+4
+15
+49
+9
+0
+15
+2
+11
+I
+6
+I
+0
+I
+0
+I
+0
+n
+p
+12
+s
+9
+test/unit
+x
+7
+require
+s
+7
+coderay
+x
+12
+WordListTest
+x
+4
+Test
+n
+x
+4
+Unit
+x
+8
+TestCase
+x
+10
+open_class
+x
+14
+__class_init__
+M
+1
+n
+n
+x
+12
+WordListTest
+i
+232
+5
+66
+5
+45
+0
+1
+47
+49
+2
+1
+15
+65
+7
+3
+7
+4
+64
+7
+5
+64
+7
+6
+64
+7
+7
+64
+7
+8
+64
+7
+9
+64
+7
+10
+64
+7
+11
+64
+35
+8
+49
+12
+2
+15
+65
+7
+13
+7
+14
+64
+7
+15
+64
+7
+16
+64
+7
+17
+64
+7
+18
+64
+7
+11
+64
+35
+6
+49
+12
+2
+15
+65
+7
+19
+7
+20
+64
+7
+21
+64
+7
+11
+64
+35
+3
+49
+12
+2
+15
+65
+7
+22
+45
+23
+24
+13
+71
+25
+47
+9
+115
+47
+49
+26
+0
+13
+7
+27
+47
+49
+28
+1
+15
+8
+120
+7
+27
+49
+25
+1
+45
+3
+29
+7
+30
+49
+31
+2
+45
+13
+32
+7
+33
+49
+31
+2
+45
+19
+34
+7
+35
+49
+31
+2
+49
+12
+2
+15
+99
+7
+36
+7
+37
+65
+67
+49
+38
+0
+49
+39
+4
+15
+99
+7
+40
+7
+41
+65
+67
+49
+38
+0
+49
+39
+4
+15
+99
+7
+42
+7
+43
+65
+67
+49
+38
+0
+49
+39
+4
+15
+99
+7
+44
+7
+45
+65
+67
+49
+38
+0
+49
+39
+4
+15
+99
+7
+46
+7
+47
+65
+67
+49
+38
+0
+49
+39
+4
+15
+99
+7
+48
+7
+49
+65
+67
+49
+38
+0
+49
+39
+4
+11
+I
+a
+I
+0
+I
+0
+I
+0
+n
+p
+50
+x
+7
+CodeRay
+n
+x
+7
+include
+x
+14
+RESERVED_WORDS
+s
+3
+asm
+s
+5
+break
+s
+4
+case
+s
+8
+continue
+s
+7
+default
+s
+2
+do
+s
+4
+else
+s
+3
+...
+x
+9
+const_set
+x
+16
+PREDEFINED_TYPES
+s
+3
+int
+s
+4
+long
+s
+5
+short
+s
+4
+char
+s
+4
+void
+x
+20
+PREDEFINED_CONSTANTS
+s
+3
+EOF
+s
+4
+NULL
+x
+10
+IDENT_KIND
+x
+8
+WordList
+n
+x
+3
+new
+x
+8
+allocate
+x
+5
+ident
+x
+10
+initialize
+n
+x
+8
+reserved
+x
+3
+add
+n
+x
+8
+pre_type
+n
+x
+12
+pre_constant
+x
+22
+test_word_list_example
+M
+1
+n
+n
+x
+22
+test_word_list_example
+i
+17
+5
+7
+0
+45
+1
+2
+7
+3
+64
+49
+4
+1
+47
+49
+5
+2
+11
+I
+4
+I
+0
+I
+0
+I
+0
+n
+p
+6
+x
+8
+pre_type
+x
+10
+IDENT_KIND
+n
+s
+4
+void
+x
+2
+[]
+x
+12
+assert_equal
+p
+5
+I
+0
+I
+1d
+I
+0
+I
+1e
+I
+11
+x
+59
+/Users/murphy/ruby/coderay-0.9/test/functional/word_list.rb
+p
+0
+x
+17
+method_visibility
+x
+15
+add_defn_method
+x
+14
+test_word_list
+M
+1
+n
+n
+x
+14
+test_word_list
+i
+73
+45
+0
+1
+13
+71
+2
+47
+9
+23
+47
+49
+3
+0
+13
+7
+4
+47
+49
+5
+1
+15
+8
+28
+7
+4
+49
+2
+1
+7
+6
+64
+35
+1
+7
+7
+49
+8
+2
+19
+0
+15
+5
+7
+7
+20
+0
+7
+6
+64
+49
+9
+1
+47
+49
+10
+2
+15
+5
+7
+4
+20
+0
+7
+11
+64
+49
+9
+1
+47
+49
+10
+2
+11
+I
+5
+I
+1
+I
+0
+I
+0
+n
+p
+12
+x
+8
+WordList
+n
+x
+3
+new
+x
+8
+allocate
+x
+5
+ident
+x
+10
+initialize
+s
+6
+foobar
+x
+8
+reserved
+x
+3
+add
+x
+2
+[]
+x
+12
+assert_equal
+s
+6
+FooBar
+p
+9
+I
+0
+I
+22
+I
+0
+I
+23
+I
+29
+I
+24
+I
+39
+I
+25
+I
+49
+x
+59
+/Users/murphy/ruby/coderay-0.9/test/functional/word_list.rb
+p
+1
+x
+4
+list
+x
+21
+test_word_list_cached
+M
+1
+n
+n
+x
+21
+test_word_list_cached
+i
+75
+45
+0
+1
+13
+71
+2
+47
+9
+24
+47
+49
+3
+0
+13
+7
+4
+2
+47
+49
+5
+2
+15
+8
+30
+7
+4
+2
+49
+2
+2
+7
+6
+64
+35
+1
+7
+7
+49
+8
+2
+19
+0
+15
+5
+7
+7
+20
+0
+7
+6
+64
+49
+9
+1
+47
+49
+10
+2
+15
+5
+7
+4
+20
+0
+7
+11
+64
+49
+9
+1
+47
+49
+10
+2
+11
+I
+5
+I
+1
+I
+0
+I
+0
+n
+p
+12
+x
+8
+WordList
+n
+x
+3
+new
+x
+8
+allocate
+x
+5
+ident
+x
+10
+initialize
+s
+6
+foobar
+x
+8
+reserved
+x
+3
+add
+x
+2
+[]
+x
+12
+assert_equal
+s
+6
+FooBar
+p
+9
+I
+0
+I
+28
+I
+0
+I
+29
+I
+2b
+I
+2a
+I
+3b
+I
+2b
+I
+4b
+x
+59
+/Users/murphy/ruby/coderay-0.9/test/functional/word_list.rb
+p
+1
+x
+4
+list
+x
+28
+test_case_ignoring_word_list
+M
+1
+n
+n
+x
+28
+test_case_ignoring_word_list
+i
+178
+45
+0
+1
+13
+71
+2
+47
+9
+23
+47
+49
+3
+0
+13
+7
+4
+47
+49
+5
+1
+15
+8
+28
+7
+4
+49
+2
+1
+7
+6
+64
+35
+1
+7
+7
+49
+8
+2
+19
+0
+15
+5
+7
+4
+20
+0
+7
+9
+64
+49
+10
+1
+47
+49
+11
+2
+15
+5
+7
+7
+20
+0
+7
+6
+64
+49
+10
+1
+47
+49
+11
+2
+15
+5
+7
+7
+20
+0
+7
+12
+64
+49
+10
+1
+47
+49
+11
+2
+15
+45
+0
+13
+13
+71
+2
+47
+9
+112
+47
+49
+3
+0
+13
+7
+4
+47
+49
+5
+1
+15
+8
+117
+7
+4
+49
+2
+1
+7
+12
+64
+35
+1
+7
+7
+49
+8
+2
+19
+0
+15
+5
+7
+4
+20
+0
+7
+9
+64
+49
+10
+1
+47
+49
+11
+2
+15
+5
+7
+7
+20
+0
+7
+6
+64
+49
+10
+1
+47
+49
+11
+2
+15
+5
+7
+7
+20
+0
+7
+12
+64
+49
+10
+1
+47
+49
+11
+2
+11
+I
+5
+I
+1
+I
+0
+I
+0
+n
+p
+14
+x
+20
+CaseIgnoringWordList
+n
+x
+3
+new
+x
+8
+allocate
+x
+5
+ident
+x
+10
+initialize
+s
+6
+foobar
+x
+8
+reserved
+x
+3
+add
+s
+3
+foo
+x
+2
+[]
+x
+12
+assert_equal
+s
+6
+FooBar
+n
+p
+19
+I
+0
+I
+2e
+I
+0
+I
+2f
+I
+29
+I
+30
+I
+39
+I
+31
+I
+49
+I
+32
+I
+59
+I
+34
+I
+82
+I
+35
+I
+92
+I
+36
+I
+a2
+I
+37
+I
+b2
+x
+59
+/Users/murphy/ruby/coderay-0.9/test/functional/word_list.rb
+p
+1
+x
+4
+list
+x
+35
+test_case_ignoring_word_list_cached
+M
+1
+n
+n
+x
+35
+test_case_ignoring_word_list_cached
+i
+182
+45
+0
+1
+13
+71
+2
+47
+9
+24
+47
+49
+3
+0
+13
+7
+4
+2
+47
+49
+5
+2
+15
+8
+30
+7
+4
+2
+49
+2
+2
+7
+6
+64
+35
+1
+7
+7
+49
+8
+2
+19
+0
+15
+5
+7
+4
+20
+0
+7
+9
+64
+49
+10
+1
+47
+49
+11
+2
+15
+5
+7
+7
+20
+0
+7
+6
+64
+49
+10
+1
+47
+49
+11
+2
+15
+5
+7
+7
+20
+0
+7
+12
+64
+49
+10
+1
+47
+49
+11
+2
+15
+45
+0
+13
+13
+71
+2
+47
+9
+115
+47
+49
+3
+0
+13
+7
+4
+2
+47
+49
+5
+2
+15
+8
+121
+7
+4
+2
+49
+2
+2
+7
+12
+64
+35
+1
+7
+7
+49
+8
+2
+19
+0
+15
+5
+7
+4
+20
+0
+7
+9
+64
+49
+10
+1
+47
+49
+11
+2
+15
+5
+7
+7
+20
+0
+7
+6
+64
+49
+10
+1
+47
+49
+11
+2
+15
+5
+7
+7
+20
+0
+7
+12
+64
+49
+10
+1
+47
+49
+11
+2
+11
+I
+5
+I
+1
+I
+0
+I
+0
+n
+p
+14
+x
+20
+CaseIgnoringWordList
+n
+x
+3
+new
+x
+8
+allocate
+x
+5
+ident
+x
+10
+initialize
+s
+6
+foobar
+x
+8
+reserved
+x
+3
+add
+s
+3
+foo
+x
+2
+[]
+x
+12
+assert_equal
+s
+6
+FooBar
+n
+p
+19
+I
+0
+I
+3a
+I
+0
+I
+3b
+I
+2b
+I
+3c
+I
+3b
+I
+3d
+I
+4b
+I
+3e
+I
+5b
+I
+40
+I
+86
+I
+41
+I
+96
+I
+42
+I
+a6
+I
+43
+I
+b6
+x
+59
+/Users/murphy/ruby/coderay-0.9/test/functional/word_list.rb
+p
+1
+x
+4
+list
+x
+8
+test_dup
+M
+1
+n
+n
+x
+8
+test_dup
+i
+110
+45
+0
+1
+13
+71
+2
+47
+9
+23
+47
+49
+3
+0
+13
+7
+4
+47
+49
+5
+1
+15
+8
+28
+7
+4
+49
+2
+1
+7
+6
+64
+35
+1
+7
+7
+49
+8
+2
+19
+0
+15
+5
+7
+7
+20
+0
+7
+6
+64
+49
+9
+1
+47
+49
+10
+2
+15
+20
+0
+49
+11
+0
+19
+1
+15
+20
+1
+7
+6
+64
+35
+1
+7
+12
+49
+8
+2
+15
+5
+7
+12
+20
+1
+7
+6
+64
+49
+9
+1
+47
+49
+10
+2
+15
+5
+7
+7
+20
+0
+7
+6
+64
+49
+9
+1
+47
+49
+10
+2
+11
+I
+6
+I
+2
+I
+0
+I
+0
+n
+p
+13
+x
+8
+WordList
+n
+x
+3
+new
+x
+8
+allocate
+x
+5
+ident
+x
+10
+initialize
+s
+6
+foobar
+x
+8
+reserved
+x
+3
+add
+x
+2
+[]
+x
+12
+assert_equal
+x
+3
+dup
+x
+7
+keyword
+p
+15
+I
+0
+I
+46
+I
+0
+I
+47
+I
+29
+I
+48
+I
+39
+I
+49
+I
+41
+I
+4a
+I
+4e
+I
+4b
+I
+5e
+I
+4c
+I
+6e
+x
+59
+/Users/murphy/ruby/coderay-0.9/test/functional/word_list.rb
+p
+2
+x
+4
+list
+x
+5
+list2
+p
+45
+I
+2
+I
+6
+I
+b
+I
+9
+I
+e
+I
+a
+I
+23
+I
+b
+I
+2c
+I
+e
+I
+2f
+I
+f
+I
+3e
+I
+10
+I
+47
+I
+13
+I
+4a
+I
+14
+I
+59
+I
+18
+I
+78
+I
+19
+I
+7d
+I
+18
+I
+80
+I
+1a
+I
+85
+I
+18
+I
+88
+I
+1b
+I
+8d
+I
+18
+I
+94
+I
+1d
+I
+a2
+I
+22
+I
+b0
+I
+28
+I
+be
+I
+2e
+I
+cc
+I
+3a
+I
+da
+I
+46
+I
+e8
+x
+59
+/Users/murphy/ruby/coderay-0.9/test/functional/word_list.rb
+p
+0
+x
+13
+attach_method
+p
+7
+I
+0
+I
+1
+I
+9
+I
+2
+I
+12
+I
+4
+I
+35
+x
+59
+/Users/murphy/ruby/coderay-0.9/test/functional/word_list.rb
+p
+0
+++ /dev/null
- GNU LESSER GENERAL PUBLIC LICENSE
- Version 2.1, February 1999
-
- Copyright (C) 1991, 1999 Free Software Foundation, Inc.
- 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
- Everyone is permitted to copy and distribute verbatim copies
- of this license document, but changing it is not allowed.
-
-[This is the first released version of the Lesser GPL. It also counts
- as the successor of the GNU Library Public License, version 2, hence
- the version number 2.1.]
-
- Preamble
-
- The licenses for most software are designed to take away your
-freedom to share and change it. By contrast, the GNU General Public
-Licenses are intended to guarantee your freedom to share and change
-free software--to make sure the software is free for all its users.
-
- This license, the Lesser General Public License, applies to some
-specially designated software packages--typically libraries--of the
-Free Software Foundation and other authors who decide to use it. You
-can use it too, but we suggest you first think carefully about whether
-this license or the ordinary General Public License is the better
-strategy to use in any particular case, based on the explanations below.
-
- When we speak of free software, we are referring to freedom of use,
-not price. Our General Public Licenses are designed to make sure that
-you have the freedom to distribute copies of free software (and charge
-for this service if you wish); that you receive source code or can get
-it if you want it; that you can change the software and use pieces of
-it in new free programs; and that you are informed that you can do
-these things.
-
- To protect your rights, we need to make restrictions that forbid
-distributors to deny you these rights or to ask you to surrender these
-rights. These restrictions translate to certain responsibilities for
-you if you distribute copies of the library or if you modify it.
-
- For example, if you distribute copies of the library, whether gratis
-or for a fee, you must give the recipients all the rights that we gave
-you. You must make sure that they, too, receive or can get the source
-code. If you link other code with the library, you must provide
-complete object files to the recipients, so that they can relink them
-with the library after making changes to the library and recompiling
-it. And you must show them these terms so they know their rights.
-
- We protect your rights with a two-step method: (1) we copyright the
-library, and (2) we offer you this license, which gives you legal
-permission to copy, distribute and/or modify the library.
-
- To protect each distributor, we want to make it very clear that
-there is no warranty for the free library. Also, if the library is
-modified by someone else and passed on, the recipients should know
-that what they have is not the original version, so that the original
-author's reputation will not be affected by problems that might be
-introduced by others.
-\f
- Finally, software patents pose a constant threat to the existence of
-any free program. We wish to make sure that a company cannot
-effectively restrict the users of a free program by obtaining a
-restrictive license from a patent holder. Therefore, we insist that
-any patent license obtained for a version of the library must be
-consistent with the full freedom of use specified in this license.
-
- Most GNU software, including some libraries, is covered by the
-ordinary GNU General Public License. This license, the GNU Lesser
-General Public License, applies to certain designated libraries, and
-is quite different from the ordinary General Public License. We use
-this license for certain libraries in order to permit linking those
-libraries into non-free programs.
-
- When a program is linked with a library, whether statically or using
-a shared library, the combination of the two is legally speaking a
-combined work, a derivative of the original library. The ordinary
-General Public License therefore permits such linking only if the
-entire combination fits its criteria of freedom. The Lesser General
-Public License permits more lax criteria for linking other code with
-the library.
-
- We call this license the "Lesser" General Public License because it
-does Less to protect the user's freedom than the ordinary General
-Public License. It also provides other free software developers Less
-of an advantage over competing non-free programs. These disadvantages
-are the reason we use the ordinary General Public License for many
-libraries. However, the Lesser license provides advantages in certain
-special circumstances.
-
- For example, on rare occasions, there may be a special need to
-encourage the widest possible use of a certain library, so that it becomes
-a de-facto standard. To achieve this, non-free programs must be
-allowed to use the library. A more frequent case is that a free
-library does the same job as widely used non-free libraries. In this
-case, there is little to gain by limiting the free library to free
-software only, so we use the Lesser General Public License.
-
- In other cases, permission to use a particular library in non-free
-programs enables a greater number of people to use a large body of
-free software. For example, permission to use the GNU C Library in
-non-free programs enables many more people to use the whole GNU
-operating system, as well as its variant, the GNU/Linux operating
-system.
-
- Although the Lesser General Public License is Less protective of the
-users' freedom, it does ensure that the user of a program that is
-linked with the Library has the freedom and the wherewithal to run
-that program using a modified version of the Library.
-
- The precise terms and conditions for copying, distribution and
-modification follow. Pay close attention to the difference between a
-"work based on the library" and a "work that uses the library". The
-former contains code derived from the library, whereas the latter must
-be combined with the library in order to run.
-\f
- GNU LESSER GENERAL PUBLIC LICENSE
- TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
-
- 0. This License Agreement applies to any software library or other
-program which contains a notice placed by the copyright holder or
-other authorized party saying it may be distributed under the terms of
-this Lesser General Public License (also called "this License").
-Each licensee is addressed as "you".
-
- A "library" means a collection of software functions and/or data
-prepared so as to be conveniently linked with application programs
-(which use some of those functions and data) to form executables.
-
- The "Library", below, refers to any such software library or work
-which has been distributed under these terms. A "work based on the
-Library" means either the Library or any derivative work under
-copyright law: that is to say, a work containing the Library or a
-portion of it, either verbatim or with modifications and/or translated
-straightforwardly into another language. (Hereinafter, translation is
-included without limitation in the term "modification".)
-
- "Source code" for a work means the preferred form of the work for
-making modifications to it. For a library, complete source code means
-all the source code for all modules it contains, plus any associated
-interface definition files, plus the scripts used to control compilation
-and installation of the library.
-
- Activities other than copying, distribution and modification are not
-covered by this License; they are outside its scope. The act of
-running a program using the Library is not restricted, and output from
-such a program is covered only if its contents constitute a work based
-on the Library (independent of the use of the Library in a tool for
-writing it). Whether that is true depends on what the Library does
-and what the program that uses the Library does.
-
- 1. You may copy and distribute verbatim copies of the Library's
-complete source code as you receive it, in any medium, provided that
-you conspicuously and appropriately publish on each copy an
-appropriate copyright notice and disclaimer of warranty; keep intact
-all the notices that refer to this License and to the absence of any
-warranty; and distribute a copy of this License along with the
-Library.
-
- You may charge a fee for the physical act of transferring a copy,
-and you may at your option offer warranty protection in exchange for a
-fee.
-\f
- 2. You may modify your copy or copies of the Library or any portion
-of it, thus forming a work based on the Library, and copy and
-distribute such modifications or work under the terms of Section 1
-above, provided that you also meet all of these conditions:
-
- a) The modified work must itself be a software library.
-
- b) You must cause the files modified to carry prominent notices
- stating that you changed the files and the date of any change.
-
- c) You must cause the whole of the work to be licensed at no
- charge to all third parties under the terms of this License.
-
- d) If a facility in the modified Library refers to a function or a
- table of data to be supplied by an application program that uses
- the facility, other than as an argument passed when the facility
- is invoked, then you must make a good faith effort to ensure that,
- in the event an application does not supply such function or
- table, the facility still operates, and performs whatever part of
- its purpose remains meaningful.
-
- (For example, a function in a library to compute square roots has
- a purpose that is entirely well-defined independent of the
- application. Therefore, Subsection 2d requires that any
- application-supplied function or table used by this function must
- be optional: if the application does not supply it, the square
- root function must still compute square roots.)
-
-These requirements apply to the modified work as a whole. If
-identifiable sections of that work are not derived from the Library,
-and can be reasonably considered independent and separate works in
-themselves, then this License, and its terms, do not apply to those
-sections when you distribute them as separate works. But when you
-distribute the same sections as part of a whole which is a work based
-on the Library, the distribution of the whole must be on the terms of
-this License, whose permissions for other licensees extend to the
-entire whole, and thus to each and every part regardless of who wrote
-it.
-
-Thus, it is not the intent of this section to claim rights or contest
-your rights to work written entirely by you; rather, the intent is to
-exercise the right to control the distribution of derivative or
-collective works based on the Library.
-
-In addition, mere aggregation of another work not based on the Library
-with the Library (or with a work based on the Library) on a volume of
-a storage or distribution medium does not bring the other work under
-the scope of this License.
-
- 3. You may opt to apply the terms of the ordinary GNU General Public
-License instead of this License to a given copy of the Library. To do
-this, you must alter all the notices that refer to this License, so
-that they refer to the ordinary GNU General Public License, version 2,
-instead of to this License. (If a newer version than version 2 of the
-ordinary GNU General Public License has appeared, then you can specify
-that version instead if you wish.) Do not make any other change in
-these notices.
-\f
- Once this change is made in a given copy, it is irreversible for
-that copy, so the ordinary GNU General Public License applies to all
-subsequent copies and derivative works made from that copy.
-
- This option is useful when you wish to copy part of the code of
-the Library into a program that is not a library.
-
- 4. You may copy and distribute the Library (or a portion or
-derivative of it, under Section 2) in object code or executable form
-under the terms of Sections 1 and 2 above provided that you accompany
-it with the complete corresponding machine-readable source code, which
-must be distributed under the terms of Sections 1 and 2 above on a
-medium customarily used for software interchange.
-
- If distribution of object code is made by offering access to copy
-from a designated place, then offering equivalent access to copy the
-source code from the same place satisfies the requirement to
-distribute the source code, even though third parties are not
-compelled to copy the source along with the object code.
-
- 5. A program that contains no derivative of any portion of the
-Library, but is designed to work with the Library by being compiled or
-linked with it, is called a "work that uses the Library". Such a
-work, in isolation, is not a derivative work of the Library, and
-therefore falls outside the scope of this License.
-
- However, linking a "work that uses the Library" with the Library
-creates an executable that is a derivative of the Library (because it
-contains portions of the Library), rather than a "work that uses the
-library". The executable is therefore covered by this License.
-Section 6 states terms for distribution of such executables.
-
- When a "work that uses the Library" uses material from a header file
-that is part of the Library, the object code for the work may be a
-derivative work of the Library even though the source code is not.
-Whether this is true is especially significant if the work can be
-linked without the Library, or if the work is itself a library. The
-threshold for this to be true is not precisely defined by law.
-
- If such an object file uses only numerical parameters, data
-structure layouts and accessors, and small macros and small inline
-functions (ten lines or less in length), then the use of the object
-file is unrestricted, regardless of whether it is legally a derivative
-work. (Executables containing this object code plus portions of the
-Library will still fall under Section 6.)
-
- Otherwise, if the work is a derivative of the Library, you may
-distribute the object code for the work under the terms of Section 6.
-Any executables containing that work also fall under Section 6,
-whether or not they are linked directly with the Library itself.
-\f
- 6. As an exception to the Sections above, you may also combine or
-link a "work that uses the Library" with the Library to produce a
-work containing portions of the Library, and distribute that work
-under terms of your choice, provided that the terms permit
-modification of the work for the customer's own use and reverse
-engineering for debugging such modifications.
-
- You must give prominent notice with each copy of the work that the
-Library is used in it and that the Library and its use are covered by
-this License. You must supply a copy of this License. If the work
-during execution displays copyright notices, you must include the
-copyright notice for the Library among them, as well as a reference
-directing the user to the copy of this License. Also, you must do one
-of these things:
-
- a) Accompany the work with the complete corresponding
- machine-readable source code for the Library including whatever
- changes were used in the work (which must be distributed under
- Sections 1 and 2 above); and, if the work is an executable linked
- with the Library, with the complete machine-readable "work that
- uses the Library", as object code and/or source code, so that the
- user can modify the Library and then relink to produce a modified
- executable containing the modified Library. (It is understood
- that the user who changes the contents of definitions files in the
- Library will not necessarily be able to recompile the application
- to use the modified definitions.)
-
- b) Use a suitable shared library mechanism for linking with the
- Library. A suitable mechanism is one that (1) uses at run time a
- copy of the library already present on the user's computer system,
- rather than copying library functions into the executable, and (2)
- will operate properly with a modified version of the library, if
- the user installs one, as long as the modified version is
- interface-compatible with the version that the work was made with.
-
- c) Accompany the work with a written offer, valid for at
- least three years, to give the same user the materials
- specified in Subsection 6a, above, for a charge no more
- than the cost of performing this distribution.
-
- d) If distribution of the work is made by offering access to copy
- from a designated place, offer equivalent access to copy the above
- specified materials from the same place.
-
- e) Verify that the user has already received a copy of these
- materials or that you have already sent this user a copy.
-
- For an executable, the required form of the "work that uses the
-Library" must include any data and utility programs needed for
-reproducing the executable from it. However, as a special exception,
-the materials to be distributed need not include anything that is
-normally distributed (in either source or binary form) with the major
-components (compiler, kernel, and so on) of the operating system on
-which the executable runs, unless that component itself accompanies
-the executable.
-
- It may happen that this requirement contradicts the license
-restrictions of other proprietary libraries that do not normally
-accompany the operating system. Such a contradiction means you cannot
-use both them and the Library together in an executable that you
-distribute.
-\f
- 7. You may place library facilities that are a work based on the
-Library side-by-side in a single library together with other library
-facilities not covered by this License, and distribute such a combined
-library, provided that the separate distribution of the work based on
-the Library and of the other library facilities is otherwise
-permitted, and provided that you do these two things:
-
- a) Accompany the combined library with a copy of the same work
- based on the Library, uncombined with any other library
- facilities. This must be distributed under the terms of the
- Sections above.
-
- b) Give prominent notice with the combined library of the fact
- that part of it is a work based on the Library, and explaining
- where to find the accompanying uncombined form of the same work.
-
- 8. You may not copy, modify, sublicense, link with, or distribute
-the Library except as expressly provided under this License. Any
-attempt otherwise to copy, modify, sublicense, link with, or
-distribute the Library is void, and will automatically terminate your
-rights under this License. However, parties who have received copies,
-or rights, from you under this License will not have their licenses
-terminated so long as such parties remain in full compliance.
-
- 9. You are not required to accept this License, since you have not
-signed it. However, nothing else grants you permission to modify or
-distribute the Library or its derivative works. These actions are
-prohibited by law if you do not accept this License. Therefore, by
-modifying or distributing the Library (or any work based on the
-Library), you indicate your acceptance of this License to do so, and
-all its terms and conditions for copying, distributing or modifying
-the Library or works based on it.
-
- 10. Each time you redistribute the Library (or any work based on the
-Library), the recipient automatically receives a license from the
-original licensor to copy, distribute, link with or modify the Library
-subject to these terms and conditions. You may not impose any further
-restrictions on the recipients' exercise of the rights granted herein.
-You are not responsible for enforcing compliance by third parties with
-this License.
-\f
- 11. If, as a consequence of a court judgment or allegation of patent
-infringement or for any other reason (not limited to patent issues),
-conditions are imposed on you (whether by court order, agreement or
-otherwise) that contradict the conditions of this License, they do not
-excuse you from the conditions of this License. If you cannot
-distribute so as to satisfy simultaneously your obligations under this
-License and any other pertinent obligations, then as a consequence you
-may not distribute the Library at all. For example, if a patent
-license would not permit royalty-free redistribution of the Library by
-all those who receive copies directly or indirectly through you, then
-the only way you could satisfy both it and this License would be to
-refrain entirely from distribution of the Library.
-
-If any portion of this section is held invalid or unenforceable under any
-particular circumstance, the balance of the section is intended to apply,
-and the section as a whole is intended to apply in other circumstances.
-
-It is not the purpose of this section to induce you to infringe any
-patents or other property right claims or to contest validity of any
-such claims; this section has the sole purpose of protecting the
-integrity of the free software distribution system which is
-implemented by public license practices. Many people have made
-generous contributions to the wide range of software distributed
-through that system in reliance on consistent application of that
-system; it is up to the author/donor to decide if he or she is willing
-to distribute software through any other system and a licensee cannot
-impose that choice.
-
-This section is intended to make thoroughly clear what is believed to
-be a consequence of the rest of this License.
-
- 12. If the distribution and/or use of the Library is restricted in
-certain countries either by patents or by copyrighted interfaces, the
-original copyright holder who places the Library under this License may add
-an explicit geographical distribution limitation excluding those countries,
-so that distribution is permitted only in or among countries not thus
-excluded. In such case, this License incorporates the limitation as if
-written in the body of this License.
-
- 13. The Free Software Foundation may publish revised and/or new
-versions of the Lesser General Public License from time to time.
-Such new versions will be similar in spirit to the present version,
-but may differ in detail to address new problems or concerns.
-
-Each version is given a distinguishing version number. If the Library
-specifies a version number of this License which applies to it and
-"any later version", you have the option of following the terms and
-conditions either of that version or of any later version published by
-the Free Software Foundation. If the Library does not specify a
-license version number, you may choose any version ever published by
-the Free Software Foundation.
-\f
- 14. If you wish to incorporate parts of the Library into other free
-programs whose distribution conditions are incompatible with these,
-write to the author to ask for permission. For software which is
-copyrighted by the Free Software Foundation, write to the Free
-Software Foundation; we sometimes make exceptions for this. Our
-decision will be guided by the two goals of preserving the free status
-of all derivatives of our free software and of promoting the sharing
-and reuse of software generally.
-
- NO WARRANTY
-
- 15. BECAUSE THE LIBRARY IS LICENSED FREE OF CHARGE, THERE IS NO
-WARRANTY FOR THE LIBRARY, TO THE EXTENT PERMITTED BY APPLICABLE LAW.
-EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR
-OTHER PARTIES PROVIDE THE LIBRARY "AS IS" WITHOUT WARRANTY OF ANY
-KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE
-IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
-PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE
-LIBRARY IS WITH YOU. SHOULD THE LIBRARY PROVE DEFECTIVE, YOU ASSUME
-THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
-
- 16. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN
-WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY
-AND/OR REDISTRIBUTE THE LIBRARY AS PERMITTED ABOVE, BE LIABLE TO YOU
-FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR
-CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE
-LIBRARY (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING
-RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A
-FAILURE OF THE LIBRARY TO OPERATE WITH ANY OTHER SOFTWARE), EVEN IF
-SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH
-DAMAGES.
-
- END OF TERMS AND CONDITIONS
-\f
- How to Apply These Terms to Your New Libraries
-
- If you develop a new library, and you want it to be of the greatest
-possible use to the public, we recommend making it free software that
-everyone can redistribute and change. You can do so by permitting
-redistribution under these terms (or, alternatively, under the terms of the
-ordinary General Public License).
-
- To apply these terms, attach the following notices to the library. It is
-safest to attach them to the start of each source file to most effectively
-convey the exclusion of warranty; and each file should have at least the
-"copyright" line and a pointer to where the full notice is found.
-
- <one line to give the library's name and a brief idea of what it does.>
- Copyright (C) <year> <name of author>
-
- This library is free software; you can redistribute it and/or
- modify it under the terms of the GNU Lesser General Public
- License as published by the Free Software Foundation; either
- version 2.1 of the License, or (at your option) any later version.
-
- This library is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- Lesser General Public License for more details.
-
- You should have received a copy of the GNU Lesser General Public
- License along with this library; if not, write to the Free Software
- Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
-
-Also add information on how to contact you by electronic and paper mail.
-
-You should also get your employer (if you work as a programmer) or your
-school, if any, to sign a "copyright disclaimer" for the library, if
-necessary. Here is a sample; alter the names:
-
- Yoyodyne, Inc., hereby disclaims all copyright interest in the
- library `Frob' (a library for tweaking knobs) written by James Random Hacker.
-
- <signature of Ty Coon>, 1 April 1990
- Ty Coon, President of Vice
-
-That's all there is to it!
-
-
+++ /dev/null
-= CodeRay
-
-[- Tired of blue'n'gray? Try the original version of this documentation on
-coderay.rubychan.de[http://coderay.rubychan.de/doc/] (use Ctrl+Click to open it in its own frame.) -]
-
-== About
-CodeRay is a Ruby library for syntax highlighting.
-
-Syntax highlighting means: You put your code in, and you get it back colored;
-Keywords, strings, floats, comments - all in different colors.
-And with line numbers.
-
-*Syntax* *Highlighting*...
-* makes code easier to read and maintain
-* lets you detect syntax errors faster
-* helps you to understand the syntax of a language
-* looks nice
-* is what everybody should have on their website
-* solves all your problems and makes the girls run after you
-
-Version: 0.9.2
-Author:: murphy (Kornelius Kalnbach)
-Contact:: murphy rubychan de
-Website:: coderay.rubychan.de[http://coderay.rubychan.de]
-License:: GNU LGPL; see LICENSE file in the main directory.
-
-== Installation
-
-You need RubyGems[http://rubyforge.org/frs/?group_id=126].
-
- % gem install coderay
-
-
-=== Dependencies
-
-CodeRay needs Ruby 1.8.6 or later. It also runs with Ruby 1.9.1+ and JRuby 1.1+.
-
-
-== Example Usage
-(Forgive me, but this is not highlighted.)
-
- require 'coderay'
-
- tokens = CodeRay.scan "puts 'Hello, world!'", :ruby
- page = tokens.html :line_numbers => :inline, :wrap => :page
- puts page
-
-
-== Documentation
-
-See CodeRay.
-
-Please report errors in this documentation to <murphy rubychan de>.
-
-
-== Credits
-
-=== Special Thanks to
-
-* licenser (Heinz N. Gies) for ending my QBasic career, inventing the Coder
- project and the input/output plugin system.
- CodeRay would not exist without him.
-* bovi (Daniel Bovensiepen) for helping me out on various occasions.
-
-=== Thanks to
-
-* Caleb Clausen for writing RubyLexer (see
- http://rubyforge.org/projects/rubylexer) and lots of very interesting mail
- traffic
-* birkenfeld (Georg Brandl) and mitsuhiku (Arnim Ronacher) for PyKleur, now pygments.
- You guys rock!
-* Jamis Buck for writing Syntax (see http://rubyforge.org/projects/syntax)
- I got some useful ideas from it.
-* Doug Kearns and everyone else who worked on ruby.vim - it not only helped me
- coding CodeRay, but also gave me a wonderful target to reach for the Ruby
- scanner.
-* everyone who uses CodeBB on http://www.rubyforen.de and http://www.python-forum.de
-* iGEL, magichisoka, manveru, WoNáDo and everyone I forgot from rubyforen.de
-* Dethix from ruby-mine.de
-* zickzackw
-* Dookie (who is no longer with us...) and Leonidas from http://www.python-forum.de
-* Andreas Schwarz for finding out that CaseIgnoringWordList was not case
- ignoring! Such things really make you write tests.
-* closure for the first version of the Scheme scanner.
-* Stefan Walk for the first version of the JavaScript and PHP scanners.
-* Josh Goebel for another version of the JavaScript scanner, a SQL and a Diff scanner.
-* Jonathan Younger for pointing out the licence confusion caused by wrong LICENSE file.
-* Jeremy Hinegardner for finding the shebang-on-empty-file bug in FileType.
-* Charles Oliver Nutter and Yehuda Katz for helping me benchmark CodeRay on JRuby.
-* Andreas Neuhaus for pointing out a markup bug in coderay/for_redcloth.
-* 0xf30fc7 for the FileType patch concerning Delphi file extensions.
-* The folks at redmine.org - thank you for using and fixing CodeRay!
-* Keith Pitt for his SQL scanners
-* Rob Aldred for the terminal encoder
-* Trans for pointing out $DEBUG dependencies
-* Flameeyes for finding that Term::ANSIColor was obsolete
-* matz and all Ruby gods and gurus
-* The inventors of: the computer, the internet, the true color display, HTML &
- CSS, VIM, Ruby, pizza, microwaves, guitars, scouting, programming, anime,
- manga, coke and green ice tea.
-
-Where would we be without all those people?
-
-=== Created using
-
-* Ruby[http://ruby-lang.org/]
-* Chihiro (my Sony VAIO laptop); Henrietta (my old MacBook);
- Triella, born Rico (my new MacBook); as well as
- Seras and Hikari (my PCs)
-* RDE[http://homepage2.nifty.com/sakazuki/rde_e.html],
- VIM[http://vim.org] and TextMate[http://macromates.com]
-* Subversion[http://subversion.tigris.org/]
-* Redmine[http://redmine.org/]
-* Firefox[http://www.mozilla.org/products/firefox/],
- Firebug[http://getfirebug.com/], Safari[http://www.apple.com/safari/], and
- Thunderbird[http://www.mozilla.org/products/thunderbird/]
-* RubyGems[http://docs.rubygems.org/] and Rake[http://rake.rubyforge.org/]
-* TortoiseSVN[http://tortoisesvn.tigris.org/] using Apache via
- XAMPP[http://www.apachefriends.org/en/xampp.html]
-* RDoc (though I'm quite unsatisfied with it)
-* Microsoft Windows (yes, I confess!) and MacOS X
-* GNUWin32, MinGW and some other tools to make the shell under windows a bit
- less useless
-* Term::ANSIColor[http://term-ansicolor.rubyforge.org/]
-* PLEAC[http://pleac.sourceforge.net/] code examples
-
-=== Free
-
-* As you can see, CodeRay was created under heavy use of *free* software.
-* So CodeRay is also *free*.
-* If you use CodeRay to create software, think about making this software
- *free*, too.
-* Thanks :)
+++ /dev/null
-# = CodeRay Library
-#
-# CodeRay is a Ruby library for syntax highlighting.
-#
-# I try to make CodeRay easy to use and intuitive, but at the same time fully featured, complete,
-# fast and efficient.
-#
-# See README.
-#
-# It consists mainly of
-# * the main engine: CodeRay (Scanners::Scanner, Tokens/TokenStream, Encoders::Encoder), PluginHost
-# * the scanners in CodeRay::Scanners
-# * the encoders in CodeRay::Encoders
-#
-# Here's a fancy graphic to light up this gray docu:
-#
-# http://cycnus.de/raindark/coderay/scheme.png
-#
-# == Documentation
-#
-# See CodeRay, Encoders, Scanners, Tokens.
-#
-# == Usage
-#
-# Remember you need RubyGems to use CodeRay, unless you have it in your load path. Run Ruby with
-# -rubygems option if required.
-#
-# === Highlight Ruby code in a string as html
-#
-# require 'coderay'
-# print CodeRay.scan('puts "Hello, world!"', :ruby).html
-#
-# # prints something like this:
-# puts <span class="s">"Hello, world!"</span>
-#
-#
-# === Highlight C code from a file in a html div
-#
-# require 'coderay'
-# print CodeRay.scan(File.read('ruby.h'), :c).div
-# print CodeRay.scan_file('ruby.h').html.div
-#
-# You can include this div in your page. The used CSS styles can be printed with
-#
-# % coderay_stylesheet
-#
-# === Highlight without typing too much
-#
-# If you are one of the hasty (or lazy, or extremely curious) people, just run this file:
-#
-# % ruby -rubygems /path/to/coderay/coderay.rb > example.html
-#
-# and look at the file it created in your browser.
-#
-# = CodeRay Module
-#
-# The CodeRay module provides convenience methods for the engine.
-#
-# * The +lang+ and +format+ arguments select Scanner and Encoder to use. These are
-# simply lower-case symbols, like <tt>:python</tt> or <tt>:html</tt>.
-# * All methods take an optional hash as last parameter, +options+, that is send to
-# the Encoder / Scanner.
-# * Input and language are always sorted in this order: +code+, +lang+.
-# (This is in alphabetical order, if you need a mnemonic ;)
-#
-# You should be able to highlight everything you want just using these methods;
-# so there is no need to dive into CodeRay's deep class hierarchy.
-#
-# The examples in the demo directory demonstrate common cases using this interface.
-#
-# = Basic Access Ways
-#
-# Read this to get a general view what CodeRay provides.
-#
-# == Scanning
-#
-# Scanning means analysing an input string, splitting it up into Tokens.
-# Each Token knows about what type it is: string, comment, class name, etc.
-#
-# Each +lang+ (language) has its own Scanner; for example, <tt>:ruby</tt> code is
-# handled by CodeRay::Scanners::Ruby.
-#
-# CodeRay.scan:: Scan a string in a given language into Tokens.
-# This is the most common method to use.
-# CodeRay.scan_file:: Scan a file and guess the language using FileType.
-#
-# The Tokens object you get from these methods can encode itself; see Tokens.
-#
-# == Encoding
-#
-# Encoding means compiling Tokens into an output. This can be colored HTML or
-# LaTeX, a textual statistic or just the number of non-whitespace tokens.
-#
-# Each Encoder provides output in a specific +format+, so you select Encoders via
-# formats like <tt>:html</tt> or <tt>:statistic</tt>.
-#
-# CodeRay.encode:: Scan and encode a string in a given language.
-# CodeRay.encode_tokens:: Encode the given tokens.
-# CodeRay.encode_file:: Scan a file, guess the language using FileType and encode it.
-#
-# == Streaming
-#
-# Streaming saves RAM by running Scanner and Encoder in some sort of
-# pipe mode; see TokenStream.
-#
-# CodeRay.scan_stream:: Scan in stream mode.
-#
-# == All-in-One Encoding
-#
-# CodeRay.encode:: Highlight a string with a given input and output format.
-#
-# == Instanciating
-#
-# You can use an Encoder instance to highlight multiple inputs. This way, the setup
-# for this Encoder must only be done once.
-#
-# CodeRay.encoder:: Create an Encoder instance with format and options.
-# CodeRay.scanner:: Create an Scanner instance for lang, with '' as default code.
-#
-# To make use of CodeRay.scanner, use CodeRay::Scanner::code=.
-#
-# The scanning methods provide more flexibility; we recommend to use these.
-#
-# == Reusing Scanners and Encoders
-#
-# If you want to re-use scanners and encoders (because that is faster), see
-# CodeRay::Duo for the most convenient (and recommended) interface.
-module CodeRay
-
- $CODERAY_DEBUG ||= false
-
- # Version: Major.Minor.Teeny[.Revision]
- # Major: 0 for pre-stable, 1 for stable
- # Minor: feature milestone
- # Teeny: development state, 0 for pre-release
- # Revision: Subversion Revision number (generated on rake gem:make)
- VERSION = '0.9.2'
-
- require 'coderay/tokens'
- require 'coderay/token_classes'
- require 'coderay/scanner'
- require 'coderay/encoder'
- require 'coderay/duo'
- require 'coderay/style'
-
-
- class << self
-
- # Scans the given +code+ (a String) with the Scanner for +lang+.
- #
- # This is a simple way to use CodeRay. Example:
- # require 'coderay'
- # page = CodeRay.scan("puts 'Hello, world!'", :ruby).html
- #
- # See also demo/demo_simple.
- def scan code, lang, options = {}, &block
- scanner = Scanners[lang].new code, options, &block
- scanner.tokenize
- end
-
- # Scans +filename+ (a path to a code file) with the Scanner for +lang+.
- #
- # If +lang+ is :auto or omitted, the CodeRay::FileType module is used to
- # determine it. If it cannot find out what type it is, it uses
- # CodeRay::Scanners::Plaintext.
- #
- # Calls CodeRay.scan.
- #
- # Example:
- # require 'coderay'
- # page = CodeRay.scan_file('some_c_code.c').html
- def scan_file filename, lang = :auto, options = {}, &block
- file = IO.read filename
- if lang == :auto
- require 'coderay/helpers/file_type'
- lang = FileType.fetch filename, :plaintext, true
- end
- scan file, lang, options = {}, &block
- end
-
- # Scan the +code+ (a string) with the scanner for +lang+.
- #
- # Calls scan.
- #
- # See CodeRay.scan.
- def scan_stream code, lang, options = {}, &block
- options[:stream] = true
- scan code, lang, options, &block
- end
-
- # Encode a string in Streaming mode.
- #
- # This starts scanning +code+ with the the Scanner for +lang+
- # while encodes the output with the Encoder for +format+.
- # +options+ will be passed to the Encoder.
- #
- # See CodeRay::Encoder.encode_stream
- def encode_stream code, lang, format, options = {}
- encoder(format, options).encode_stream code, lang, options
- end
-
- # Encode a string.
- #
- # This scans +code+ with the the Scanner for +lang+ and then
- # encodes it with the Encoder for +format+.
- # +options+ will be passed to the Encoder.
- #
- # See CodeRay::Encoder.encode
- def encode code, lang, format, options = {}
- encoder(format, options).encode code, lang, options
- end
-
- # Highlight a string into a HTML <div>.
- #
- # CSS styles use classes, so you have to include a stylesheet
- # in your output.
- #
- # See encode.
- def highlight code, lang, options = { :css => :class }, format = :div
- encode code, lang, format, options
- end
-
- # Encode pre-scanned Tokens.
- # Use this together with CodeRay.scan:
- #
- # require 'coderay'
- #
- # # Highlight a short Ruby code example in a HTML span
- # tokens = CodeRay.scan '1 + 2', :ruby
- # puts CodeRay.encode_tokens(tokens, :span)
- #
- def encode_tokens tokens, format, options = {}
- encoder(format, options).encode_tokens tokens, options
- end
-
- # Encodes +filename+ (a path to a code file) with the Scanner for +lang+.
- #
- # See CodeRay.scan_file.
- # Notice that the second argument is the output +format+, not the input language.
- #
- # Example:
- # require 'coderay'
- # page = CodeRay.encode_file 'some_c_code.c', :html
- def encode_file filename, format, options = {}
- tokens = scan_file filename, :auto, get_scanner_options(options)
- encode_tokens tokens, format, options
- end
-
- # Highlight a file into a HTML <div>.
- #
- # CSS styles use classes, so you have to include a stylesheet
- # in your output.
- #
- # See encode.
- def highlight_file filename, options = { :css => :class }, format = :div
- encode_file filename, format, options
- end
-
- # Finds the Encoder class for +format+ and creates an instance, passing
- # +options+ to it.
- #
- # Example:
- # require 'coderay'
- #
- # stats = CodeRay.encoder(:statistic)
- # stats.encode("puts 17 + 4\n", :ruby)
- #
- # puts '%d out of %d tokens have the kind :integer.' % [
- # stats.type_stats[:integer].count,
- # stats.real_token_count
- # ]
- # #-> 2 out of 4 tokens have the kind :integer.
- def encoder format, options = {}
- Encoders[format].new options
- end
-
- # Finds the Scanner class for +lang+ and creates an instance, passing
- # +options+ to it.
- #
- # See Scanner.new.
- def scanner lang, options = {}
- Scanners[lang].new '', options
- end
-
- # Extract the options for the scanner from the +options+ hash.
- #
- # Returns an empty Hash if <tt>:scanner_options</tt> is not set.
- #
- # This is used if a method like CodeRay.encode has to provide options
- # for Encoder _and_ scanner.
- def get_scanner_options options
- options.fetch :scanner_options, {}
- end
-
- end
-
- # This Exception is raised when you try to stream with something that is not
- # capable of streaming.
- class NotStreamableError < Exception
- def initialize obj
- @obj = obj
- end
-
- def to_s
- '%s is not Streamable!' % @obj.class
- end
- end
-
- # A dummy module that is included by subclasses of CodeRay::Scanner an CodeRay::Encoder
- # to show that they are able to handle streams.
- module Streamable
- end
-
-end
-
-# Run a test script.
-if $0 == __FILE__
- $stderr.print 'Press key to print demo.'; gets
- # Just use this file as an example of Ruby code.
- code = File.read(__FILE__)[/module CodeRay.*/m]
- print CodeRay.scan(code, :ruby).html
-end
+++ /dev/null
-module CodeRay
-
- # = Duo
- #
- # A Duo is a convenient way to use CodeRay. You just create a Duo,
- # giving it a lang (language of the input code) and a format (desired
- # output format), and call Duo#highlight with the code.
- #
- # Duo makes it easy to re-use both scanner and encoder for a repetitive
- # task. It also provides a very easy interface syntax:
- #
- # require 'coderay'
- # CodeRay::Duo[:python, :div].highlight 'import this'
- #
- # Until you want to do uncommon things with CodeRay, I recommend to use
- # this method, since it takes care of everything.
- class Duo
-
- attr_accessor :lang, :format, :options
-
- # Create a new Duo, holding a lang and a format to highlight code.
- #
- # simple:
- # CodeRay::Duo[:ruby, :page].highlight 'bla 42'
- #
- # streaming:
- # CodeRay::Duo[:ruby, :page].highlight 'bar 23', :stream => true
- #
- # with options:
- # CodeRay::Duo[:ruby, :html, :hint => :debug].highlight '????::??'
- #
- # alternative syntax without options:
- # CodeRay::Duo[:ruby => :statistic].encode 'class << self; end'
- #
- # alternative syntax with options:
- # CodeRay::Duo[{ :ruby => :statistic }, :do => :something].encode 'abc'
- #
- # The options are forwarded to scanner and encoder
- # (see CodeRay.get_scanner_options).
- def initialize lang = nil, format = nil, options = {}
- if format == nil and lang.is_a? Hash and lang.size == 1
- @lang = lang.keys.first
- @format = lang[@lang]
- else
- @lang = lang
- @format = format
- end
- @options = options
- end
-
- class << self
- # To allow calls like Duo[:ruby, :html].highlight.
- alias [] new
- end
-
- # The scanner of the duo. Only created once.
- def scanner
- @scanner ||= CodeRay.scanner @lang, CodeRay.get_scanner_options(@options)
- end
-
- # The encoder of the duo. Only created once.
- def encoder
- @encoder ||= CodeRay.encoder @format, @options
- end
-
- # Tokenize and highlight the code using +scanner+ and +encoder+.
- #
- # If the :stream option is set, the Duo will go into streaming mode,
- # saving memory for the cost of time.
- def encode code, options = { :stream => false }
- stream = options.delete :stream
- options = @options.merge options
- if stream
- encoder.encode_stream(code, @lang, options)
- else
- scanner.code = code
- encoder.encode_tokens(scanner.tokenize, options)
- end
- end
- alias highlight encode
-
- end
-
-end
-
+++ /dev/null
-module CodeRay
-
- # This module holds the Encoder class and its subclasses.
- # For example, the HTML encoder is named CodeRay::Encoders::HTML
- # can be found in coderay/encoders/html.
- #
- # Encoders also provides methods and constants for the register
- # mechanism and the [] method that returns the Encoder class
- # belonging to the given format.
- module Encoders
- extend PluginHost
- plugin_path File.dirname(__FILE__), 'encoders'
-
- # = Encoder
- #
- # The Encoder base class. Together with Scanner and
- # Tokens, it forms the highlighting triad.
- #
- # Encoder instances take a Tokens object and do something with it.
- #
- # The most common Encoder is surely the HTML encoder
- # (CodeRay::Encoders::HTML). It highlights the code in a colorful
- # html page.
- # If you want the highlighted code in a div or a span instead,
- # use its subclasses Div and Span.
- class Encoder
- extend Plugin
- plugin_host Encoders
-
- attr_reader :token_stream
-
- class << self
-
- # Returns if the Encoder can be used in streaming mode.
- def streamable?
- is_a? Streamable
- end
-
- # If FILE_EXTENSION isn't defined, this method returns the
- # downcase class name instead.
- def const_missing sym
- if sym == :FILE_EXTENSION
- plugin_id
- else
- super
- end
- end
-
- end
-
- # Subclasses are to store their default options in this constant.
- DEFAULT_OPTIONS = { :stream => false }
-
- # The options you gave the Encoder at creating.
- attr_accessor :options
-
- # Creates a new Encoder.
- # +options+ is saved and used for all encode operations, as long
- # as you don't overwrite it there by passing additional options.
- #
- # Encoder objects provide three encode methods:
- # - encode simply takes a +code+ string and a +lang+
- # - encode_tokens expects a +tokens+ object instead
- # - encode_stream is like encode, but uses streaming mode.
- #
- # Each method has an optional +options+ parameter. These are
- # added to the options you passed at creation.
- def initialize options = {}
- @options = self.class::DEFAULT_OPTIONS.merge options
- raise "I am only the basic Encoder class. I can't encode "\
- "anything. :( Use my subclasses." if self.class == Encoder
- end
-
- # Encode a Tokens object.
- def encode_tokens tokens, options = {}
- options = @options.merge options
- setup options
- compile tokens, options
- finish options
- end
-
- # Encode the given +code+ after tokenizing it using the Scanner
- # for +lang+.
- def encode code, lang, options = {}
- options = @options.merge options
- scanner_options = CodeRay.get_scanner_options(options)
- tokens = CodeRay.scan code, lang, scanner_options
- encode_tokens tokens, options
- end
-
- # You can use highlight instead of encode, if that seems
- # more clear to you.
- alias highlight encode
-
- # Encode the given +code+ using the Scanner for +lang+ in
- # streaming mode.
- def encode_stream code, lang, options = {}
- raise NotStreamableError, self unless kind_of? Streamable
- options = @options.merge options
- setup options
- scanner_options = CodeRay.get_scanner_options options
- @token_stream =
- CodeRay.scan_stream code, lang, scanner_options, &self
- finish options
- end
-
- # Behave like a proc. The token method is converted to a proc.
- def to_proc
- method(:token).to_proc
- end
-
- # Return the default file extension for outputs of this encoder.
- def file_extension
- self.class::FILE_EXTENSION
- end
-
- protected
-
- # Called with merged options before encoding starts.
- # Sets @out to an empty string.
- #
- # See the HTML Encoder for an example of option caching.
- def setup options
- @out = ''
- end
-
- # Called with +content+ and +kind+ of the currently scanned token.
- # For simple scanners, it's enougth to implement this method.
- #
- # By default, it calls text_token or block_token, depending on
- # whether +content+ is a String.
- def token content, kind
- encoded_token =
- if content.is_a? ::String
- text_token content, kind
- elsif content.is_a? ::Symbol
- block_token content, kind
- else
- raise 'Unknown token content type: %p' % [content]
- end
- append_encoded_token_to_output encoded_token
- end
-
- def append_encoded_token_to_output encoded_token
- @out << encoded_token if encoded_token && defined?(@out) && @out
- end
-
- # Called for each text token ([text, kind]), where text is a String.
- def text_token text, kind
- end
-
- # Called for each block (non-text) token ([action, kind]),
- # where +action+ is a Symbol.
- #
- # Calls open_token, close_token, begin_line, and end_line according to
- # the value of +action+.
- def block_token action, kind
- case action
- when :open
- open_token kind
- when :close
- close_token kind
- when :begin_line
- begin_line kind
- when :end_line
- end_line kind
- else
- raise 'unknown block action: %p' % action
- end
- end
-
- # Called for each block token at the start of the block ([:open, kind]).
- def open_token kind
- end
-
- # Called for each block token end of the block ([:close, kind]).
- def close_token kind
- end
-
- # Called for each line token block at the start of the line ([:begin_line, kind]).
- def begin_line kind
- end
-
- # Called for each line token block at the end of the line ([:end_line, kind]).
- def end_line kind
- end
-
- # Called with merged options after encoding starts.
- # The return value is the result of encoding, typically @out.
- def finish options
- @out
- end
-
- # Do the encoding.
- #
- # The already created +tokens+ object must be used; it can be a
- # TokenStream or a Tokens object.
- if RUBY_VERSION >= '1.9'
- def compile tokens, options
- for text, kind in tokens
- token text, kind
- end
- end
- else
- def compile tokens, options
- tokens.each(&self)
- end
- end
-
- end
-
- end
-end
+++ /dev/null
-module CodeRay
-module Encoders
-
- map \
- :loc => :lines_of_code,
- :plain => :text,
- :stats => :statistic,
- :terminal => :term,
- :tex => :latex
-
-end
-end
+++ /dev/null
-($:.unshift '../..'; require 'coderay') unless defined? CodeRay
-module CodeRay
-module Encoders
-
- load :token_class_filter
-
- class CommentFilter < TokenClassFilter
-
- register_for :comment_filter
-
- DEFAULT_OPTIONS = superclass::DEFAULT_OPTIONS.merge \
- :exclude => [:comment]
-
- end
-
-end
-end
-
-if $0 == __FILE__
- $VERBOSE = true
- $: << File.join(File.dirname(__FILE__), '..')
- eval DATA.read, nil, $0, __LINE__ + 4
-end
-
-__END__
-require 'test/unit'
-
-class CommentFilterTest < Test::Unit::TestCase
-
- def test_filtering_comments
- tokens = CodeRay.scan <<-RUBY, :ruby
-#!/usr/bin/env ruby
-# a minimal Ruby program
-puts "Hello world!"
- RUBY
- assert_equal <<-RUBY_FILTERED, tokens.comment_filter.text
-#!/usr/bin/env ruby
-
-puts "Hello world!"
- RUBY_FILTERED
- end
-
-end
\ No newline at end of file
+++ /dev/null
-module CodeRay
-module Encoders
-
- class Count < Encoder
-
- include Streamable
- register_for :count
-
- protected
-
- def setup options
- @out = 0
- end
-
- def token text, kind
- @out += 1
- end
- end
-
-end
-end
+++ /dev/null
-module CodeRay
-module Encoders
-
- # = Debug Encoder
- #
- # Fast encoder producing simple debug output.
- #
- # It is readable and diff-able and is used for testing.
- #
- # You cannot fully restore the tokens information from the
- # output, because consecutive :space tokens are merged.
- # Use Tokens#dump for caching purposes.
- class Debug < Encoder
-
- include Streamable
- register_for :debug
-
- FILE_EXTENSION = 'raydebug'
-
- protected
- def text_token text, kind
- if kind == :space
- text
- else
- text = text.gsub(/[)\\]/, '\\\\\0') # escape ) and \
- "#{kind}(#{text})"
- end
- end
-
- def open_token kind
- "#{kind}<"
- end
-
- def close_token kind
- ">"
- end
-
- def begin_line kind
- "#{kind}["
- end
-
- def end_line kind
- "]"
- end
-
- end
-
-end
-end
+++ /dev/null
-module CodeRay
-module Encoders
-
- load :html
-
- class Div < HTML
-
- FILE_EXTENSION = 'div.html'
-
- register_for :div
-
- DEFAULT_OPTIONS = HTML::DEFAULT_OPTIONS.merge \
- :css => :style,
- :wrap => :div
-
- end
-
-end
-end
+++ /dev/null
-($:.unshift '../..'; require 'coderay') unless defined? CodeRay
-module CodeRay
-module Encoders
-
- class Filter < Encoder
-
- register_for :filter
-
- protected
- def setup options
- @out = Tokens.new
- end
-
- def text_token text, kind
- [text, kind] if include_text_token? text, kind
- end
-
- def include_text_token? text, kind
- true
- end
-
- def block_token action, kind
- [action, kind] if include_block_token? action, kind
- end
-
- def include_block_token? action, kind
- true
- end
-
- end
-
-end
-end
-
-if $0 == __FILE__
- $VERBOSE = true
- $: << File.join(File.dirname(__FILE__), '..')
- eval DATA.read, nil, $0, __LINE__ + 4
-end
-
-__END__
-require 'test/unit'
-
-class FilterTest < Test::Unit::TestCase
-
- def test_creation
- assert CodeRay::Encoders::Filter < CodeRay::Encoders::Encoder
- filter = nil
- assert_nothing_raised do
- filter = CodeRay.encoder :filter
- end
- assert_kind_of CodeRay::Encoders::Encoder, filter
- end
-
- def test_filtering_text_tokens
- tokens = CodeRay::Tokens.new
- 10.times do |i|
- tokens << [i.to_s, :index]
- end
- assert_equal tokens, CodeRay::Encoders::Filter.new.encode_tokens(tokens)
- assert_equal tokens, tokens.filter
- end
-
- def test_filtering_block_tokens
- tokens = CodeRay::Tokens.new
- 10.times do |i|
- tokens << [:open, :index]
- tokens << [i.to_s, :content]
- tokens << [:close, :index]
- end
- assert_equal tokens, CodeRay::Encoders::Filter.new.encode_tokens(tokens)
- assert_equal tokens, tokens.filter
- end
-
-end
+++ /dev/null
-require 'set'
-
-module CodeRay
-module Encoders
-
- # = HTML Encoder
- #
- # This is CodeRay's most important highlighter:
- # It provides save, fast XHTML generation and CSS support.
- #
- # == Usage
- #
- # require 'coderay'
- # puts CodeRay.scan('Some /code/', :ruby).html #-> a HTML page
- # puts CodeRay.scan('Some /code/', :ruby).html(:wrap => :span)
- # #-> <span class="CodeRay"><span class="co">Some</span> /code/</span>
- # puts CodeRay.scan('Some /code/', :ruby).span #-> the same
- #
- # puts CodeRay.scan('Some code', :ruby).html(
- # :wrap => nil,
- # :line_numbers => :inline,
- # :css => :style
- # )
- # #-> <span class="no">1</span> <span style="color:#036; font-weight:bold;">Some</span> code
- #
- # == Options
- #
- # === :tab_width
- # Convert \t characters to +n+ spaces (a number.)
- # Default: 8
- #
- # === :css
- # How to include the styles; can be :class or :style.
- #
- # Default: :class
- #
- # === :wrap
- # Wrap in :page, :div, :span or nil.
- #
- # You can also use Encoders::Div and Encoders::Span.
- #
- # Default: nil
- #
- # === :title
- #
- # The title of the HTML page (works only when :wrap is set to :page.)
- #
- # Default: 'CodeRay output'
- #
- # === :line_numbers
- # Include line numbers in :table, :inline, :list or nil (no line numbers)
- #
- # Default: nil
- #
- # === :line_number_start
- # Where to start with line number counting.
- #
- # Default: 1
- #
- # === :bold_every
- # Make every +n+-th number appear bold.
- #
- # Default: 10
- #
- # === :highlight_lines
- #
- # Highlights certain line numbers.
- # Can be any Enumerable, typically just an Array or Range, of numbers.
- #
- # Bolding is deactivated when :highlight_lines is set. It only makes sense
- # in combination with :line_numbers.
- #
- # Default: nil
- #
- # === :hint
- # Include some information into the output using the title attribute.
- # Can be :info (show token type on mouse-over), :info_long (with full path)
- # or :debug (via inspect).
- #
- # Default: false
- class HTML < Encoder
-
- include Streamable
- register_for :html
-
- FILE_EXTENSION = 'html'
-
- DEFAULT_OPTIONS = {
- :tab_width => 8,
-
- :css => :class,
-
- :style => :cycnus,
- :wrap => nil,
- :title => 'CodeRay output',
-
- :line_numbers => nil,
- :line_number_start => 1,
- :bold_every => 10,
- :highlight_lines => nil,
-
- :hint => false,
- }
-
- helper :output, :css
-
- attr_reader :css
-
- protected
-
- HTML_ESCAPE = { #:nodoc:
- '&' => '&',
- '"' => '"',
- '>' => '>',
- '<' => '<',
- }
-
- # This was to prevent illegal HTML.
- # Strange chars should still be avoided in codes.
- evil_chars = Array(0x00...0x20) - [?\n, ?\t, ?\s]
- evil_chars.each { |i| HTML_ESCAPE[i.chr] = ' ' }
- #ansi_chars = Array(0x7f..0xff)
- #ansi_chars.each { |i| HTML_ESCAPE[i.chr] = '&#%d;' % i }
- # \x9 (\t) and \xA (\n) not included
- #HTML_ESCAPE_PATTERN = /[\t&"><\0-\x8\xB-\x1f\x7f-\xff]/
- HTML_ESCAPE_PATTERN = /[\t"&><\0-\x8\xB-\x1f]/
-
- TOKEN_KIND_TO_INFO = Hash.new { |h, kind|
- h[kind] =
- case kind
- when :pre_constant
- 'Predefined constant'
- else
- kind.to_s.gsub(/_/, ' ').gsub(/\b\w/) { $&.capitalize }
- end
- }
-
- TRANSPARENT_TOKEN_KINDS = [
- :delimiter, :modifier, :content, :escape, :inline_delimiter,
- ].to_set
-
- # Generate a hint about the given +classes+ in a +hint+ style.
- #
- # +hint+ may be :info, :info_long or :debug.
- def self.token_path_to_hint hint, classes
- title =
- case hint
- when :info
- TOKEN_KIND_TO_INFO[classes.first]
- when :info_long
- classes.reverse.map { |kind| TOKEN_KIND_TO_INFO[kind] }.join('/')
- when :debug
- classes.inspect
- end
- title ? " title=\"#{title}\"" : ''
- end
-
- def setup options
- super
-
- @HTML_ESCAPE = HTML_ESCAPE.dup
- @HTML_ESCAPE["\t"] = ' ' * options[:tab_width]
-
- @opened = [nil]
- @css = CSS.new options[:style]
-
- hint = options[:hint]
- if hint and not [:debug, :info, :info_long].include? hint
- raise ArgumentError, "Unknown value %p for :hint; \
- expected :info, :debug, false, or nil." % hint
- end
-
- case options[:css]
-
- when :class
- @css_style = Hash.new do |h, k|
- c = CodeRay::Tokens::ClassOfKind[k.first]
- if c == :NO_HIGHLIGHT and not hint
- h[k.dup] = false
- else
- title = if hint
- HTML.token_path_to_hint(hint, k[1..-1] << k.first)
- else
- ''
- end
- if c == :NO_HIGHLIGHT
- h[k.dup] = '<span%s>' % [title]
- else
- h[k.dup] = '<span%s class="%s">' % [title, c]
- end
- end
- end
-
- when :style
- @css_style = Hash.new do |h, k|
- if k.is_a? ::Array
- styles = k.dup
- else
- styles = [k]
- end
- type = styles.first
- classes = styles.map { |c| Tokens::ClassOfKind[c] }
- if classes.first == :NO_HIGHLIGHT and not hint
- h[k] = false
- else
- styles.shift if TRANSPARENT_TOKEN_KINDS.include? styles.first
- title = HTML.token_path_to_hint hint, styles
- style = @css[*classes]
- h[k] =
- if style
- '<span%s style="%s">' % [title, style]
- else
- false
- end
- end
- end
-
- else
- raise ArgumentError, "Unknown value %p for :css." % options[:css]
-
- end
- end
-
- def finish options
- not_needed = @opened.shift
- @out << '</span>' * @opened.size
- unless @opened.empty?
- warn '%d tokens still open: %p' % [@opened.size, @opened]
- end
-
- @out.extend Output
- @out.css = @css
- @out.numerize! options[:line_numbers], options
- @out.wrap! options[:wrap]
- @out.apply_title! options[:title]
-
- super
- end
-
- def token text, type = :plain
- case text
-
- when nil
- # raise 'Token with nil as text was given: %p' % [[text, type]]
-
- when String
- if text =~ /#{HTML_ESCAPE_PATTERN}/o
- text = text.gsub(/#{HTML_ESCAPE_PATTERN}/o) { |m| @HTML_ESCAPE[m] }
- end
- @opened[0] = type
- if text != "\n" && style = @css_style[@opened]
- @out << style << text << '</span>'
- else
- @out << text
- end
-
-
- # token groups, eg. strings
- when :open
- @opened[0] = type
- @out << (@css_style[@opened] || '<span>')
- @opened << type
- when :close
- if @opened.empty?
- # nothing to close
- else
- if $CODERAY_DEBUG and (@opened.size == 1 or @opened.last != type)
- raise 'Malformed token stream: Trying to close a token (%p) \
- that is not open. Open are: %p.' % [type, @opened[1..-1]]
- end
- @out << '</span>'
- @opened.pop
- end
-
- # whole lines to be highlighted, eg. a deleted line in a diff
- when :begin_line
- @opened[0] = type
- if style = @css_style[@opened]
- @out << style.sub('<span', '<div')
- else
- @out << '<div>'
- end
- @opened << type
- when :end_line
- if @opened.empty?
- # nothing to close
- else
- if $CODERAY_DEBUG and (@opened.size == 1 or @opened.last != type)
- raise 'Malformed token stream: Trying to close a line (%p) \
- that is not open. Open are: %p.' % [type, @opened[1..-1]]
- end
- @out << '</div>'
- @opened.pop
- end
-
- else
- raise 'unknown token kind: %p' % [text]
-
- end
- end
-
- end
-
-end
-end
+++ /dev/null
-module CodeRay
-module Encoders
-
- class HTML
- class CSS
-
- attr :stylesheet
-
- def CSS.load_stylesheet style = nil
- CodeRay::Styles[style]
- end
-
- def initialize style = :default
- @classes = Hash.new
- style = CSS.load_stylesheet style
- @stylesheet = [
- style::CSS_MAIN_STYLES,
- style::TOKEN_COLORS.gsub(/^(?!$)/, '.CodeRay ')
- ].join("\n")
- parse style::TOKEN_COLORS
- end
-
- def [] *styles
- cl = @classes[styles.first]
- return '' unless cl
- style = ''
- 1.upto(styles.size) do |offset|
- break if style = cl[styles[offset .. -1]]
- end
- # warn 'Style not found: %p' % [styles] if style.empty?
- return style
- end
-
- private
-
- CSS_CLASS_PATTERN = /
- ( # $1 = selectors
- (?:
- (?: \s* \. [-\w]+ )+
- \s* ,?
- )+
- )
- \s* \{ \s*
- ( [^\}]+ )? # $2 = style
- \s* \} \s*
- |
- ( . ) # $3 = error
- /mx
- def parse stylesheet
- stylesheet.scan CSS_CLASS_PATTERN do |selectors, style, error|
- raise "CSS parse error: '#{error.inspect}' not recognized" if error
- for selector in selectors.split(',')
- classes = selector.scan(/[-\w]+/)
- cl = classes.pop
- @classes[cl] ||= Hash.new
- @classes[cl][classes] = style.to_s.strip.delete(' ').chomp(';')
- end
- end
- end
-
- end
- end
-
-end
-end
-
-if $0 == __FILE__
- require 'pp'
- pp CodeRay::Encoders::HTML::CSS.new
-end
+++ /dev/null
-module CodeRay
-module Encoders
-
- class HTML
-
- module Output
-
- def numerize *args
- clone.numerize!(*args)
- end
-
-=begin NUMERIZABLE_WRAPPINGS = {
- :table => [:div, :page, nil],
- :inline => :all,
- :list => [:div, :page, nil]
- }
- NUMERIZABLE_WRAPPINGS.default = :all
-=end
- def numerize! mode = :table, options = {}
- return self unless mode
-
- options = DEFAULT_OPTIONS.merge options
-
- start = options[:line_number_start]
- unless start.is_a? Integer
- raise ArgumentError, "Invalid value %p for :line_number_start; Integer expected." % start
- end
-
- #allowed_wrappings = NUMERIZABLE_WRAPPINGS[mode]
- #unless allowed_wrappings == :all or allowed_wrappings.include? options[:wrap]
- # raise ArgumentError, "Can't numerize, :wrap must be in %p, but is %p" % [NUMERIZABLE_WRAPPINGS, options[:wrap]]
- #end
-
- bold_every = options[:bold_every]
- highlight_lines = options[:highlight_lines]
- bolding =
- if bold_every == false && highlight_lines == nil
- proc { |line| line.to_s }
- elsif highlight_lines.is_a? Enumerable
- highlight_lines = highlight_lines.to_set
- proc do |line|
- if highlight_lines.include? line
- "<strong class=\"highlighted\">#{line}</strong>" # highlighted line numbers in bold
- else
- line.to_s
- end
- end
- elsif bold_every.is_a? Integer
- raise ArgumentError, ":bolding can't be 0." if bold_every == 0
- proc do |line|
- if line % bold_every == 0
- "<strong>#{line}</strong>" # every bold_every-th number in bold
- else
- line.to_s
- end
- end
- else
- raise ArgumentError, 'Invalid value %p for :bolding; false or Integer expected.' % bold_every
- end
-
- case mode
- when :inline
- max_width = (start + line_count).to_s.size
- line_number = start
- gsub!(/^/) do
- line_number_text = bolding.call line_number
- indent = ' ' * (max_width - line_number.to_s.size) # TODO: Optimize (10^x)
- res = "<span class=\"no\">#{indent}#{line_number_text}</span> "
- line_number += 1
- res
- end
-
- when :table
- # This is really ugly.
- # Because even monospace fonts seem to have different heights when bold,
- # I make the newline bold, both in the code and the line numbers.
- # FIXME Still not working perfect for Mr. Internet Exploder
- line_numbers = (start ... start + line_count).to_a.map(&bolding).join("\n")
- line_numbers << "\n" # also for Mr. MS Internet Exploder :-/
- line_numbers.gsub!(/\n/) { "<tt>\n</tt>" }
-
- line_numbers_table_tpl = TABLE.apply('LINE_NUMBERS', line_numbers)
- gsub!(/<\/div>\n/) { '</div>' }
- gsub!(/\n/) { "<tt>\n</tt>" }
- wrap_in! line_numbers_table_tpl
- @wrapped_in = :div
-
- when :list
- opened_tags = []
- gsub!(/^.*$\n?/) do |line|
- line.chomp!
-
- open = opened_tags.join
- line.scan(%r!<(/)?span[^>]*>?!) do |close,|
- if close
- opened_tags.pop
- else
- opened_tags << $&
- end
- end
- close = '</span>' * opened_tags.size
-
- "<li>#{open}#{line}#{close}</li>\n"
- end
- chomp!("\n")
- wrap_in! LIST
- @wrapped_in = :div
-
- else
- raise ArgumentError, 'Unknown value %p for mode: expected one of %p' %
- [mode, [:table, :list, :inline]]
- end
-
- self
- end
-
- def line_count
- line_count = count("\n")
- position_of_last_newline = rindex(?\n)
- if position_of_last_newline
- after_last_newline = self[position_of_last_newline + 1 .. -1]
- ends_with_newline = after_last_newline[/\A(?:<\/span>)*\z/]
- line_count += 1 if not ends_with_newline
- end
- line_count
- end
-
- end
-
- end
-
-end
-end
+++ /dev/null
-module CodeRay
-module Encoders
-
- class HTML
-
- # This module is included in the output String from thew HTML Encoder.
- #
- # It provides methods like wrap, div, page etc.
- #
- # Remember to use #clone instead of #dup to keep the modules the object was
- # extended with.
- #
- # TODO: more doc.
- module Output
-
- require 'coderay/encoders/html/numerization.rb'
-
- attr_accessor :css
-
- class << self
-
- # This makes Output look like a class.
- #
- # Example:
- #
- # a = Output.new '<span class="co">Code</span>'
- # a.wrap! :page
- def new string, css = CSS.new, element = nil
- output = string.clone.extend self
- output.wrapped_in = element
- output.css = css
- output
- end
-
- # Raises an exception if an object that doesn't respond to to_str is extended by Output,
- # to prevent users from misuse. Use Module#remove_method to disable.
- def extended o
- warn "The Output module is intended to extend instances of String, not #{o.class}." unless o.respond_to? :to_str
- end
-
- def make_stylesheet css, in_tag = false
- sheet = css.stylesheet
- sheet = <<-CSS if in_tag
-<style type="text/css">
-#{sheet}
-</style>
- CSS
- sheet
- end
-
- def page_template_for_css css
- sheet = make_stylesheet css
- PAGE.apply 'CSS', sheet
- end
-
- # Define a new wrapper. This is meta programming.
- def wrapper *wrappers
- wrappers.each do |wrapper|
- define_method wrapper do |*args|
- wrap wrapper, *args
- end
- define_method "#{wrapper}!".to_sym do |*args|
- wrap! wrapper, *args
- end
- end
- end
-
- end
-
- wrapper :div, :span, :page
-
- def wrapped_in? element
- wrapped_in == element
- end
-
- def wrapped_in
- @wrapped_in ||= nil
- end
- attr_writer :wrapped_in
-
- def wrap_in template
- clone.wrap_in! template
- end
-
- def wrap_in! template
- Template.wrap! self, template, 'CONTENT'
- self
- end
-
- def apply_title! title
- self.sub!(/(<title>)(<\/title>)/) { $1 + title + $2 }
- self
- end
-
- def wrap! element, *args
- return self if not element or element == wrapped_in
- case element
- when :div
- raise "Can't wrap %p in %p" % [wrapped_in, element] unless wrapped_in? nil
- wrap_in! DIV
- when :span
- raise "Can't wrap %p in %p" % [wrapped_in, element] unless wrapped_in? nil
- wrap_in! SPAN
- when :page
- wrap! :div if wrapped_in? nil
- raise "Can't wrap %p in %p" % [wrapped_in, element] unless wrapped_in? :div
- wrap_in! Output.page_template_for_css(@css)
- if args.first.is_a?(Hash) && title = args.first[:title]
- apply_title! title
- end
- self
- when nil
- return self
- else
- raise "Unknown value %p for :wrap" % element
- end
- @wrapped_in = element
- self
- end
-
- def wrap *args
- clone.wrap!(*args)
- end
-
- def stylesheet in_tag = false
- Output.make_stylesheet @css, in_tag
- end
-
- class Template < String
-
- def self.wrap! str, template, target
- target = Regexp.new(Regexp.escape("<%#{target}%>"))
- if template =~ target
- str[0,0] = $`
- str << $'
- else
- raise "Template target <%%%p%%> not found" % target
- end
- end
-
- def apply target, replacement
- target = Regexp.new(Regexp.escape("<%#{target}%>"))
- if self =~ target
- Template.new($` + replacement + $')
- else
- raise "Template target <%%%p%%> not found" % target
- end
- end
-
- module Simple
- def ` str #` <-- for stupid editors
- Template.new str
- end
- end
- end
-
- extend Template::Simple
-
-#-- don't include the templates in docu
-
- SPAN = `<span class="CodeRay"><%CONTENT%></span>`
-
- DIV = <<-`DIV`
-<div class="CodeRay">
- <div class="code"><pre><%CONTENT%></pre></div>
-</div>
- DIV
-
- TABLE = <<-`TABLE`
-<table class="CodeRay"><tr>
- <td class="line_numbers" title="click to toggle" onclick="with (this.firstChild.style) { display = (display == '') ? 'none' : '' }"><pre><%LINE_NUMBERS%></pre></td>
- <td class="code"><pre ondblclick="with (this.style) { overflow = (overflow == 'auto' || overflow == '') ? 'visible' : 'auto' }"><%CONTENT%></pre></td>
-</tr></table>
- TABLE
- # title="double click to expand"
-
- LIST = <<-`LIST`
-<ol class="CodeRay">
-<%CONTENT%>
-</ol>
- LIST
-
- PAGE = <<-`PAGE`
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
- "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
-<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="de">
-<head>
- <meta http-equiv="content-type" content="text/html; charset=utf-8" />
- <title></title>
- <style type="text/css">
-<%CSS%>
- </style>
-</head>
-<body style="background-color: white;">
-
-<%CONTENT%>
-</body>
-</html>
- PAGE
-
- end
-
- end
-
-end
-end
+++ /dev/null
-($:.unshift '../..'; require 'coderay') unless defined? CodeRay
-module CodeRay
-module Encoders
-
- # = JSON Encoder
- class JSON < Encoder
-
- register_for :json
- FILE_EXTENSION = 'json'
-
- protected
- def setup options
- begin
- require 'json'
- rescue LoadError
- require 'rubygems'
- require 'json'
- end
- @out = []
- end
-
- def text_token text, kind
- { :type => 'text', :text => text, :kind => kind }
- end
-
- def block_token action, kind
- { :type => 'block', :action => action, :kind => kind }
- end
-
- def finish options
- @out.to_json
- end
-
- end
-
-end
-end
-
-if $0 == __FILE__
- $VERBOSE = true
- $: << File.join(File.dirname(__FILE__), '..')
- eval DATA.read, nil, $0, __LINE__ + 4
-end
-
-__END__
-require 'test/unit'
-$:.delete '.'
-require 'rubygems' if RUBY_VERSION < '1.9'
-
-class JSONEncoderTest < Test::Unit::TestCase
-
- def test_json_output
- tokens = CodeRay.scan <<-RUBY, :ruby
-puts "Hello world!"
- RUBY
- require 'json'
- assert_equal [
- {"type"=>"text", "text"=>"puts", "kind"=>"ident"},
- {"type"=>"text", "text"=>" ", "kind"=>"space"},
- {"type"=>"block", "action"=>"open", "kind"=>"string"},
- {"type"=>"text", "text"=>"\"", "kind"=>"delimiter"},
- {"type"=>"text", "text"=>"Hello world!", "kind"=>"content"},
- {"type"=>"text", "text"=>"\"", "kind"=>"delimiter"},
- {"type"=>"block", "action"=>"close", "kind"=>"string"},
- {"type"=>"text", "text"=>"\n", "kind"=>"space"}
- ], JSON.load(tokens.json)
- end
-
-end
\ No newline at end of file
+++ /dev/null
-($:.unshift '../..'; require 'coderay') unless defined? CodeRay
-module CodeRay
-module Encoders
-
- # Counts the LoC (Lines of Code). Returns an Integer >= 0.
- #
- # Alias: :loc
- #
- # Everything that is not comment, markup, doctype/shebang, or an empty line,
- # is considered to be code.
- #
- # For example,
- # * HTML files not containing JavaScript have 0 LoC
- # * in a Java class without comments, LoC is the number of non-empty lines
- #
- # A Scanner class should define the token kinds that are not code in the
- # KINDS_NOT_LOC constant, which defaults to [:comment, :doctype].
- class LinesOfCode < Encoder
-
- register_for :lines_of_code
-
- NON_EMPTY_LINE = /^\s*\S.*$/
-
- def compile tokens, options
- if scanner = tokens.scanner
- kinds_not_loc = scanner.class::KINDS_NOT_LOC
- else
- warn ArgumentError, 'Tokens have no scanner.' if $VERBOSE
- kinds_not_loc = CodeRay::Scanners::Scanner::KINDS_NOT_LOC
- end
- code = tokens.token_class_filter :exclude => kinds_not_loc
- @loc = code.text.scan(NON_EMPTY_LINE).size
- end
-
- def finish options
- @loc
- end
-
- end
-
-end
-end
-
-if $0 == __FILE__
- $VERBOSE = true
- $: << File.join(File.dirname(__FILE__), '..')
- eval DATA.read, nil, $0, __LINE__ + 4
-end
-
-__END__
-require 'test/unit'
-
-class LinesOfCodeTest < Test::Unit::TestCase
-
- def test_creation
- assert CodeRay::Encoders::LinesOfCode < CodeRay::Encoders::Encoder
- filter = nil
- assert_nothing_raised do
- filter = CodeRay.encoder :loc
- end
- assert_kind_of CodeRay::Encoders::LinesOfCode, filter
- assert_nothing_raised do
- filter = CodeRay.encoder :lines_of_code
- end
- assert_kind_of CodeRay::Encoders::LinesOfCode, filter
- end
-
- def test_lines_of_code
- tokens = CodeRay.scan <<-RUBY, :ruby
-#!/usr/bin/env ruby
-
-# a minimal Ruby program
-puts "Hello world!"
- RUBY
- assert_equal 1, CodeRay::Encoders::LinesOfCode.new.encode_tokens(tokens)
- assert_equal 1, tokens.lines_of_code
- assert_equal 1, tokens.loc
- end
-
- def test_filtering_block_tokens
- tokens = CodeRay::Tokens.new
- tokens << ["Hello\n", :world]
- tokens << ["Hello\n", :space]
- tokens << ["Hello\n", :comment]
- assert_equal 2, CodeRay::Encoders::LinesOfCode.new.encode_tokens(tokens)
- assert_equal 2, tokens.lines_of_code
- assert_equal 2, tokens.loc
- end
-
-end
\ No newline at end of file
+++ /dev/null
-module CodeRay
-module Encoders
-
- # = Null Encoder
- #
- # Does nothing and returns an empty string.
- class Null < Encoder
-
- include Streamable
- register_for :null
-
- # Defined for faster processing
- def to_proc
- proc {}
- end
-
- protected
-
- def token(*)
- # do nothing
- end
-
- end
-
-end
-end
+++ /dev/null
-module CodeRay
-module Encoders
-
- load :html
-
- class Page < HTML
-
- FILE_EXTENSION = 'html'
-
- register_for :page
-
- DEFAULT_OPTIONS = HTML::DEFAULT_OPTIONS.merge \
- :css => :class,
- :wrap => :page,
- :line_numbers => :table
-
- end
-
-end
-end
+++ /dev/null
-module CodeRay
-module Encoders
-
- load :html
-
- class Span < HTML
-
- FILE_EXTENSION = 'span.html'
-
- register_for :span
-
- DEFAULT_OPTIONS = HTML::DEFAULT_OPTIONS.merge \
- :css => :style,
- :wrap => :span
-
- end
-
-end
-end
+++ /dev/null
-module CodeRay
-module Encoders
-
- # Makes a statistic for the given tokens.
- class Statistic < Encoder
-
- include Streamable
- register_for :stats, :statistic
-
- attr_reader :type_stats, :real_token_count
-
- protected
-
- TypeStats = Struct.new :count, :size
-
- def setup options
- @type_stats = Hash.new { |h, k| h[k] = TypeStats.new 0, 0 }
- @real_token_count = 0
- end
-
- def generate tokens, options
- @tokens = tokens
- super
- end
-
- def text_token text, kind
- @real_token_count += 1 unless kind == :space
- @type_stats[kind].count += 1
- @type_stats[kind].size += text.size
- @type_stats['TOTAL'].size += text.size
- @type_stats['TOTAL'].count += 1
- end
-
- # TODO Hierarchy handling
- def block_token action, kind
- @type_stats['TOTAL'].count += 1
- @type_stats['open/close'].count += 1
- end
-
- STATS = <<-STATS
-
-Code Statistics
-
-Tokens %8d
- Non-Whitespace %8d
-Bytes Total %8d
-
-Token Types (%d):
- type count ratio size (average)
--------------------------------------------------------------
-%s
- STATS
-# space 12007 33.81 % 1.7
- TOKEN_TYPES_ROW = <<-TKR
- %-20s %8d %6.2f %% %5.1f
- TKR
-
- def finish options
- all = @type_stats['TOTAL']
- all_count, all_size = all.count, all.size
- @type_stats.each do |type, stat|
- stat.size /= stat.count.to_f
- end
- types_stats = @type_stats.sort_by { |k, v| [-v.count, k.to_s] }.map do |k, v|
- TOKEN_TYPES_ROW % [k, v.count, 100.0 * v.count / all_count, v.size]
- end.join
- STATS % [
- all_count, @real_token_count, all_size,
- @type_stats.delete_if { |k, v| k.is_a? String }.size,
- types_stats
- ]
- end
-
- end
-
-end
-end
+++ /dev/null
-# encoders/term.rb
-# By Rob Aldred (http://robaldred.co.uk)
-# Based on idea by Nathan Weizenbaum (http://nex-3.com)
-# MIT License (http://www.opensource.org/licenses/mit-license.php)
-#
-# A CodeRay encoder that outputs code highlighted for a color terminal.
-# Check out http://robaldred.co.uk
-
-module CodeRay
- module Encoders
- class Term < Encoder
- register_for :term
-
- TOKEN_COLORS = {
- :annotation => '35',
- :attribute_name => '33',
- :attribute_name_fat => '33',
- :attribute_value => '31',
- :attribute_value_fat => '31',
- :bin => '1;35',
- :char => {:self => '36', :delimiter => '34'},
- :class => '1;35',
- :class_variable => '36',
- :color => '32',
- :comment => '37',
- :complex => '34',
- :constant => ['34', '4'],
- :decoration => '35',
- :definition => '1;32',
- :directive => ['32', '4'],
- :doc => '46',
- :doctype => '1;30',
- :doc_string => ['31', '4'],
- :entity => '33',
- :error => ['1;33', '41'],
- :exception => '1;31',
- :float => '1;35',
- :function => '1;34',
- :global_variable => '42',
- :hex => '1;36',
- :important => '1;31',
- :include => '33',
- :integer => '1;34',
- :interpreted => '1;35',
- :key => '35',
- :label => '1;4',
- :local_variable => '33',
- :oct => '1;35',
- :operator_name => '1;29',
- :pre_constant => '1;36',
- :pre_type => '1;30',
- :predefined => ['4', '1;34'],
- :preprocessor => '36',
- :pseudo_class => '34',
- :regexp => {
- :content => '31',
- :delimiter => '1;29',
- :modifier => '35',
- :function => '1;29'
- },
- :reserved => '1;31',
- :shell => {
- :self => '42',
- :content => '1;29',
- :delimiter => '37',
- },
- :string => {
- :self => '32',
- :modifier => '1;32',
- :escape => '1;36',
- :delimiter => '1;32',
- },
- :symbol => '1;32',
- :tag => '34',
- :tag_fat => '1;34',
- :tag_special => ['34', '4'],
- :type => '1;34',
- :value => '36',
- :variable => '34',
- :insert => '42',
- :delete => '41',
- :change => '44',
- :head => '45',
- }
- TOKEN_COLORS[:keyword] = TOKEN_COLORS[:reserved]
- TOKEN_COLORS[:method] = TOKEN_COLORS[:function]
- TOKEN_COLORS[:imaginary] = TOKEN_COLORS[:complex]
- TOKEN_COLORS[:open] = TOKEN_COLORS[:close] = TOKEN_COLORS[:nesting_delimiter] = TOKEN_COLORS[:escape] = TOKEN_COLORS[:delimiter]
-
- protected
-
- def setup(options)
- @out = ''
- @opened = [nil]
- @subcolors = nil
- end
-
- def finish(options)
- super
- end
-
- def token text, type = :plain
- case text
-
- when nil
- # raise 'Token with nil as text was given: %p' % [[text, type]]
-
- when String
-
- if color = (@subcolors || TOKEN_COLORS)[type]
- color = color[:self] || return if Hash === color
-
- @out << col(color) + text.gsub("\n", col(0) + "\n" + col(color)) + col(0)
- @out << col(@subcolors[:self]) if @subcolors && @subcolors[:self]
- else
- @out << text
- end
-
- # token groups, eg. strings
- when :open
- @opened[0] = type
- if color = TOKEN_COLORS[type]
- if Hash === color
- @subcolors = color
- @out << col(color[:self]) if color[:self]
- else
- @subcolors = {}
- @out << col(color)
- end
- end
- @opened << type
- when :close
- if @opened.empty?
- # nothing to close
- else
- @out << col(0) if (@subcolors || {})[:self]
- @subcolors = nil
- @opened.pop
- end
-
- # whole lines to be highlighted, eg. a added/modified/deleted lines in a diff
- when :begin_line
-
- when :end_line
-
- else
- raise 'unknown token kind: %p' % [text]
- end
- end
-
- private
-
- def col(color)
- Array(color).map { |c| "\e[#{c}m" }.join
- end
- end
- end
-end
\ No newline at end of file
+++ /dev/null
-module CodeRay
-module Encoders
-
- class Text < Encoder
-
- include Streamable
- register_for :text
-
- FILE_EXTENSION = 'txt'
-
- DEFAULT_OPTIONS = {
- :separator => ''
- }
-
- protected
- def setup options
- super
- @sep = options[:separator]
- end
-
- def text_token text, kind
- text + @sep
- end
-
- def finish options
- super.chomp @sep
- end
-
- end
-
-end
-end
+++ /dev/null
-($:.unshift '../..'; require 'coderay') unless defined? CodeRay
-module CodeRay
-module Encoders
-
- load :filter
-
- class TokenClassFilter < Filter
-
- include Streamable
- register_for :token_class_filter
-
- DEFAULT_OPTIONS = {
- :exclude => [],
- :include => :all
- }
-
- protected
- def setup options
- super
- @exclude = options[:exclude]
- @exclude = Array(@exclude) unless @exclude == :all
- @include = options[:include]
- @include = Array(@include) unless @include == :all
- end
-
- def include_text_token? text, kind
- (@include == :all || @include.include?(kind)) &&
- !(@exclude == :all || @exclude.include?(kind))
- end
-
- end
-
-end
-end
-
-if $0 == __FILE__
- $VERBOSE = true
- $: << File.join(File.dirname(__FILE__), '..')
- eval DATA.read, nil, $0, __LINE__ + 4
-end
-
-__END__
-require 'test/unit'
-
-class TokenClassFilterTest < Test::Unit::TestCase
-
- def test_creation
- assert CodeRay::Encoders::TokenClassFilter < CodeRay::Encoders::Encoder
- assert CodeRay::Encoders::TokenClassFilter < CodeRay::Encoders::Filter
- filter = nil
- assert_nothing_raised do
- filter = CodeRay.encoder :token_class_filter
- end
- assert_instance_of CodeRay::Encoders::TokenClassFilter, filter
- end
-
- def test_filtering_text_tokens
- tokens = CodeRay::Tokens.new
- for i in 1..10
- tokens << [i.to_s, :index]
- tokens << [' ', :space] if i < 10
- end
- assert_equal 10, CodeRay::Encoders::TokenClassFilter.new.encode_tokens(tokens, :exclude => :space).size
- assert_equal 10, tokens.token_class_filter(:exclude => :space).size
- assert_equal 9, CodeRay::Encoders::TokenClassFilter.new.encode_tokens(tokens, :include => :space).size
- assert_equal 9, tokens.token_class_filter(:include => :space).size
- assert_equal 0, CodeRay::Encoders::TokenClassFilter.new.encode_tokens(tokens, :exclude => :all).size
- assert_equal 0, tokens.token_class_filter(:exclude => :all).size
- end
-
- def test_filtering_block_tokens
- tokens = CodeRay::Tokens.new
- 10.times do |i|
- tokens << [:open, :index]
- tokens << [i.to_s, :content]
- tokens << [:close, :index]
- end
- assert_equal 20, CodeRay::Encoders::TokenClassFilter.new.encode_tokens(tokens, :include => :blubb).size
- assert_equal 20, tokens.token_class_filter(:include => :blubb).size
- assert_equal 30, CodeRay::Encoders::TokenClassFilter.new.encode_tokens(tokens, :exclude => :index).size
- assert_equal 30, tokens.token_class_filter(:exclude => :index).size
- end
-
-end
+++ /dev/null
-module CodeRay
-module Encoders
-
- # = XML Encoder
- #
- # Uses REXML. Very slow.
- class XML < Encoder
-
- include Streamable
- register_for :xml
-
- FILE_EXTENSION = 'xml'
-
- require 'rexml/document'
-
- DEFAULT_OPTIONS = {
- :tab_width => 8,
- :pretty => -1,
- :transitive => false,
- }
-
- protected
-
- def setup options
- @doc = REXML::Document.new
- @doc << REXML::XMLDecl.new
- @tab_width = options[:tab_width]
- @root = @node = @doc.add_element('coderay-tokens')
- end
-
- def finish options
- @out = ''
- @doc.write @out, options[:pretty], options[:transitive], true
- @out
- end
-
- def text_token text, kind
- if kind == :space
- token = @node
- else
- token = @node.add_element kind.to_s
- end
- text.scan(/(\x20+)|(\t+)|(\n)|[^\x20\t\n]+/) do |space, tab, nl|
- case
- when space
- token << REXML::Text.new(space, true)
- when tab
- token << REXML::Text.new(tab, true)
- when nl
- token << REXML::Text.new(nl, true)
- else
- token << REXML::Text.new($&)
- end
- end
- end
-
- def open_token kind
- @node = @node.add_element kind.to_s
- end
-
- def close_token kind
- if @node == @root
- raise 'no token to close!'
- end
- @node = @node.parent
- end
-
- end
-
-end
-end
+++ /dev/null
-module CodeRay
-module Encoders
-
- # = YAML Encoder
- #
- # Slow.
- class YAML < Encoder
-
- register_for :yaml
-
- FILE_EXTENSION = 'yaml'
-
- protected
- def compile tokens, options
- require 'yaml'
- @out = tokens.to_a.to_yaml
- end
-
- end
-
-end
-end
+++ /dev/null
-module CodeRay
-
- # A little hack to enable CodeRay highlighting in RedCloth.
- #
- # Usage:
- # require 'coderay'
- # require 'coderay/for_redcloth'
- # RedCloth.new('@[ruby]puts "Hello, World!"@').to_html
- #
- # Make sure you have RedCloth 4.0.3 activated, for example by calling
- # require 'rubygems'
- # before RedCloth is loaded and before calling CodeRay.for_redcloth.
- module ForRedCloth
-
- def self.install
- gem 'RedCloth', '>= 4.0.3' if defined? gem
- require 'redcloth'
- unless RedCloth::VERSION.to_s >= '4.0.3'
- if defined? gem
- raise 'CodeRay.for_redcloth needs RedCloth version 4.0.3 or later. ' +
- "You have #{RedCloth::VERSION}. Please gem install RedCloth."
- else
- $".delete 'redcloth.rb' # sorry, but it works
- require 'rubygems'
- return install # retry
- end
- end
- unless RedCloth::VERSION.to_s >= '4.2.2'
- warn 'CodeRay.for_redcloth works best with RedCloth version 4.2.2 or later.'
- end
- RedCloth::TextileDoc.send :include, ForRedCloth::TextileDoc
- RedCloth::Formatters::HTML.module_eval do
- def unescape(html)
- replacements = {
- '&' => '&',
- '"' => '"',
- '>' => '>',
- '<' => '<',
- }
- html.gsub(/&(?:amp|quot|[gl]t);/) { |entity| replacements[entity] }
- end
- undef code, bc_open, bc_close, escape_pre
- def code(opts) # :nodoc:
- opts[:block] = true
- if !opts[:lang] && RedCloth::VERSION.to_s >= '4.2.0'
- # simulating pre-4.2 behavior
- if opts[:text].sub!(/\A\[(\w+)\]/, '')
- if CodeRay::Scanners[$1].plugin_id == 'plaintext'
- opts[:text] = $& + opts[:text]
- else
- opts[:lang] = $1
- end
- end
- end
- if opts[:lang] && !filter_coderay
- require 'coderay'
- @in_bc ||= nil
- format = @in_bc ? :div : :span
- opts[:text] = unescape(opts[:text]) unless @in_bc
- highlighted_code = CodeRay.encode opts[:text], opts[:lang], format, :stream => true
- highlighted_code.sub!(/\A<(span|div)/) { |m| m + pba(@in_bc || opts) }
- highlighted_code
- else
- "<code#{pba(opts)}>#{opts[:text]}</code>"
- end
- end
- def bc_open(opts) # :nodoc:
- opts[:block] = true
- @in_bc = opts
- opts[:lang] ? '' : "<pre#{pba(opts)}>"
- end
- def bc_close(opts) # :nodoc:
- opts = @in_bc
- @in_bc = nil
- opts[:lang] ? '' : "</pre>\n"
- end
- def escape_pre(text)
- if @in_bc ||= nil
- text
- else
- html_esc(text, :html_escape_preformatted)
- end
- end
- end
- end
-
- module TextileDoc # :nodoc:
- attr_accessor :filter_coderay
- end
-
- end
-
-end
-
-CodeRay::ForRedCloth.install
\ No newline at end of file
+++ /dev/null
-#!/usr/bin/env ruby
-module CodeRay
-
-# = FileType
-#
-# A simple filetype recognizer.
-#
-# Copyright (c) 2006 by murphy (Kornelius Kalnbach) <murphy rubychan de>
-#
-# License:: LGPL / ask the author
-# Version:: 0.1 (2005-09-01)
-#
-# == Documentation
-#
-# # determine the type of the given
-# lang = FileType[ARGV.first]
-#
-# # return :plaintext if the file type is unknown
-# lang = FileType.fetch ARGV.first, :plaintext
-#
-# # try the shebang line, too
-# lang = FileType.fetch ARGV.first, :plaintext, true
-module FileType
-
- UnknownFileType = Class.new Exception
-
- class << self
-
- # Try to determine the file type of the file.
- #
- # +filename+ is a relative or absolute path to a file.
- #
- # The file itself is only accessed when +read_shebang+ is set to true.
- # That means you can get filetypes from files that don't exist.
- def [] filename, read_shebang = false
- name = File.basename filename
- ext = File.extname(name).sub(/^\./, '') # from last dot, delete the leading dot
- ext2 = filename.to_s[/\.(.*)/, 1] # from first dot
-
- type =
- TypeFromExt[ext] ||
- TypeFromExt[ext.downcase] ||
- (TypeFromExt[ext2] if ext2) ||
- (TypeFromExt[ext2.downcase] if ext2) ||
- TypeFromName[name] ||
- TypeFromName[name.downcase]
- type ||= shebang(filename) if read_shebang
-
- type
- end
-
- def shebang filename
- begin
- File.open filename, 'r' do |f|
- if first_line = f.gets
- if type = first_line[TypeFromShebang]
- type.to_sym
- end
- end
- end
- rescue IOError
- nil
- end
- end
-
- # This works like Hash#fetch.
- #
- # If the filetype cannot be found, the +default+ value
- # is returned.
- def fetch filename, default = nil, read_shebang = false
- if default and block_given?
- warn 'block supersedes default value argument'
- end
-
- unless type = self[filename, read_shebang]
- return yield if block_given?
- return default if default
- raise UnknownFileType, 'Could not determine type of %p.' % filename
- end
- type
- end
-
- end
-
- TypeFromExt = {
- 'c' => :c,
- 'css' => :css,
- 'diff' => :diff,
- 'dpr' => :delphi,
- 'groovy' => :groovy,
- 'gvy' => :groovy,
- 'h' => :c,
- 'htm' => :html,
- 'html' => :html,
- 'html.erb' => :rhtml,
- 'java' => :java,
- 'js' => :java_script,
- 'json' => :json,
- 'mab' => :ruby,
- 'pas' => :delphi,
- 'patch' => :diff,
- 'php' => :php,
- 'php3' => :php,
- 'php4' => :php,
- 'php5' => :php,
- 'py' => :python,
- 'py3' => :python,
- 'pyw' => :python,
- 'rake' => :ruby,
- 'raydebug' => :debug,
- 'rb' => :ruby,
- 'rbw' => :ruby,
- 'rhtml' => :rhtml,
- 'rxml' => :ruby,
- 'sch' => :scheme,
- 'sql' => :sql,
- 'ss' => :scheme,
- 'xhtml' => :xhtml,
- 'xml' => :xml,
- 'yaml' => :yaml,
- 'yml' => :yaml,
- }
- for cpp_alias in %w[cc cpp cp cxx c++ C hh hpp h++ cu]
- TypeFromExt[cpp_alias] = :cpp
- end
-
- TypeFromShebang = /\b(?:ruby|perl|python|sh)\b/
-
- TypeFromName = {
- 'Rakefile' => :ruby,
- 'Rantfile' => :ruby,
- }
-
-end
-
-end
-
-if $0 == __FILE__
- $VERBOSE = true
- eval DATA.read, nil, $0, __LINE__ + 4
-end
-
-__END__
-require 'test/unit'
-
-class FileTypeTests < Test::Unit::TestCase
-
- include CodeRay
-
- def test_fetch
- assert_raise FileType::UnknownFileType do
- FileType.fetch ''
- end
-
- assert_throws :not_found do
- FileType.fetch '.' do
- throw :not_found
- end
- end
-
- assert_equal :default, FileType.fetch('c', :default)
-
- stderr, fake_stderr = $stderr, Object.new
- $err = ''
- def fake_stderr.write x
- $err << x
- end
- $stderr = fake_stderr
- FileType.fetch('c', :default) { }
- assert_equal "block supersedes default value argument\n", $err
- $stderr = stderr
- end
-
- def test_ruby
- assert_equal :ruby, FileType['test.rb']
- assert_equal :ruby, FileType['test.java.rb']
- assert_equal :java, FileType['test.rb.java']
- assert_equal :ruby, FileType['C:\\Program Files\\x\\y\\c\\test.rbw']
- assert_equal :ruby, FileType['/usr/bin/something/Rakefile']
- assert_equal :ruby, FileType['~/myapp/gem/Rantfile']
- assert_equal :ruby, FileType['./lib/tasks\repository.rake']
- assert_not_equal :ruby, FileType['test_rb']
- assert_not_equal :ruby, FileType['Makefile']
- assert_not_equal :ruby, FileType['set.rb/set']
- assert_not_equal :ruby, FileType['~/projects/blabla/rb']
- end
-
- def test_c
- assert_equal :c, FileType['test.c']
- assert_equal :c, FileType['C:\\Program Files\\x\\y\\c\\test.h']
- assert_not_equal :c, FileType['test_c']
- assert_not_equal :c, FileType['Makefile']
- assert_not_equal :c, FileType['set.h/set']
- assert_not_equal :c, FileType['~/projects/blabla/c']
- end
-
- def test_cpp
- assert_equal :cpp, FileType['test.c++']
- assert_equal :cpp, FileType['test.cxx']
- assert_equal :cpp, FileType['test.hh']
- assert_equal :cpp, FileType['test.hpp']
- assert_equal :cpp, FileType['test.cu']
- assert_equal :cpp, FileType['test.C']
- assert_not_equal :cpp, FileType['test.c']
- assert_not_equal :cpp, FileType['test.h']
- end
-
- def test_html
- assert_equal :html, FileType['test.htm']
- assert_equal :xhtml, FileType['test.xhtml']
- assert_equal :xhtml, FileType['test.html.xhtml']
- assert_equal :rhtml, FileType['_form.rhtml']
- assert_equal :rhtml, FileType['_form.html.erb']
- end
-
- def test_yaml
- assert_equal :yaml, FileType['test.yml']
- assert_equal :yaml, FileType['test.yaml']
- assert_equal :yaml, FileType['my.html.yaml']
- assert_not_equal :yaml, FileType['YAML']
- end
-
- def test_pathname
- require 'pathname'
- pn = Pathname.new 'test.rb'
- assert_equal :ruby, FileType[pn]
- dir = Pathname.new '/etc/var/blubb'
- assert_equal :ruby, FileType[dir + pn]
- assert_equal :cpp, FileType[dir + 'test.cpp']
- end
-
- def test_no_shebang
- dir = './test'
- if File.directory? dir
- Dir.chdir dir do
- assert_equal :c, FileType['test.c']
- end
- end
- end
-
- def test_shebang_empty_file
- require 'tmpdir'
- tmpfile = File.join(Dir.tmpdir, 'bla')
- File.open(tmpfile, 'w') { } # touch
- assert_equal nil, FileType[tmpfile]
- end
-
- def test_shebang
- require 'tmpdir'
- tmpfile = File.join(Dir.tmpdir, 'bla')
- File.open(tmpfile, 'w') { |f| f.puts '#!/usr/bin/env ruby' }
- assert_equal :ruby, FileType[tmpfile, true]
- end
-
-end
+++ /dev/null
-# =GZip Simple
-#
-# A simplified interface to the gzip library +zlib+ (from the Ruby Standard Library.)
-#
-# Author: murphy (mail to murphy rubychan de)
-#
-# Version: 0.2 (2005.may.28)
-#
-# ==Documentation
-#
-# See +GZip+ module and the +String+ extensions.
-#
-module GZip
-
- require 'zlib'
-
- # The default zipping level. 7 zips good and fast.
- DEFAULT_GZIP_LEVEL = 7
-
- # Unzips the given string +s+.
- #
- # Example:
- # require 'gzip_simple'
- # print GZip.gunzip(File.read('adresses.gz'))
- def GZip.gunzip s
- Zlib::Inflate.inflate s
- end
-
- # Zips the given string +s+.
- #
- # Example:
- # require 'gzip_simple'
- # File.open('adresses.gz', 'w') do |file
- # file.write GZip.gzip('Mum: 0123 456 789', 9)
- # end
- #
- # If you provide a +level+, you can control how strong
- # the string is compressed:
- # - 0: no compression, only convert to gzip format
- # - 1: compress fast
- # - 7: compress more, but still fast (default)
- # - 8: compress more, slower
- # - 9: compress best, very slow
- def GZip.gzip s, level = DEFAULT_GZIP_LEVEL
- Zlib::Deflate.new(level).deflate s, Zlib::FINISH
- end
-end
-
-
-# String extensions to use the GZip module.
-#
-# The methods gzip and gunzip provide an even more simple
-# interface to the ZLib:
-#
-# # create a big string
-# x = 'a' * 1000
-#
-# # zip it
-# x_gz = x.gzip
-#
-# # test the result
-# puts 'Zipped %d bytes to %d bytes.' % [x.size, x_gz.size]
-# #-> Zipped 1000 bytes to 19 bytes.
-#
-# # unzipping works
-# p x_gz.gunzip == x #-> true
-class String
- # Returns the string, unzipped.
- # See GZip.gunzip
- def gunzip
- GZip.gunzip self
- end
- # Replaces the string with its unzipped value.
- # See GZip.gunzip
- def gunzip!
- replace gunzip
- end
-
- # Returns the string, zipped.
- # +level+ is the gzip compression level, see GZip.gzip.
- def gzip level = GZip::DEFAULT_GZIP_LEVEL
- GZip.gzip self, level
- end
- # Replaces the string with its zipped value.
- # See GZip.gzip.
- def gzip!(*args)
- replace gzip(*args)
- end
-end
-
-if $0 == __FILE__
- eval DATA.read, nil, $0, __LINE__+4
-end
-
-__END__
-#CODE
-
-# Testing / Benchmark
-x = 'a' * 1000
-x_gz = x.gzip
-puts 'Zipped %d bytes to %d bytes.' % [x.size, x_gz.size] #-> Zipped 1000 bytes to 19 bytes.
-p x_gz.gunzip == x #-> true
-
-require 'benchmark'
-
-INFO = 'packed to %0.3f%%' # :nodoc:
-
-x = Array.new(100000) { rand(255).chr + 'aaaaaaaaa' + rand(255).chr }.join
-Benchmark.bm(10) do |bm|
- for level in 0..9
- bm.report "zip #{level}" do
- $x = x.gzip level
- end
- puts INFO % [100.0 * $x.size / x.size]
- end
- bm.report 'zip' do
- $x = x.gzip
- end
- puts INFO % [100.0 * $x.size / x.size]
- bm.report 'unzip' do
- $x.gunzip
- end
-end
+++ /dev/null
-module CodeRay
-
-# = PluginHost
-#
-# A simple subclass plugin system.
-#
-# Example:
-# class Generators < PluginHost
-# plugin_path 'app/generators'
-# end
-#
-# class Generator
-# extend Plugin
-# PLUGIN_HOST = Generators
-# end
-#
-# class FancyGenerator < Generator
-# register_for :fancy
-# end
-#
-# Generators[:fancy] #-> FancyGenerator
-# # or
-# CodeRay.require_plugin 'Generators/fancy'
-module PluginHost
-
- # Raised if Encoders::[] fails because:
- # * a file could not be found
- # * the requested Encoder is not registered
- PluginNotFound = Class.new Exception
- HostNotFound = Class.new Exception
-
- PLUGIN_HOSTS = []
- PLUGIN_HOSTS_BY_ID = {} # dummy hash
-
- # Loads all plugins using list and load.
- def load_all
- for plugin in list
- load plugin
- end
- end
-
- # Returns the Plugin for +id+.
- #
- # Example:
- # yaml_plugin = MyPluginHost[:yaml]
- def [] id, *args, &blk
- plugin = validate_id(id)
- begin
- plugin = plugin_hash.[] plugin, *args, &blk
- end while plugin.is_a? Symbol
- plugin
- end
-
- # Alias for +[]+.
- alias load []
-
- def require_helper plugin_id, helper_name
- path = path_to File.join(plugin_id, helper_name)
- require path
- end
-
- class << self
-
- # Adds the module/class to the PLUGIN_HOSTS list.
- def extended mod
- PLUGIN_HOSTS << mod
- end
-
- # Warns you that you should not #include this module.
- def included mod
- warn "#{name} should not be included. Use extend."
- end
-
- # Find the PluginHost for host_id.
- def host_by_id host_id
- unless PLUGIN_HOSTS_BY_ID.default_proc
- ph = Hash.new do |h, a_host_id|
- for host in PLUGIN_HOSTS
- h[host.host_id] = host
- end
- h.fetch a_host_id, nil
- end
- PLUGIN_HOSTS_BY_ID.replace ph
- end
- PLUGIN_HOSTS_BY_ID[host_id]
- end
-
- end
-
- # The path where the plugins can be found.
- def plugin_path *args
- unless args.empty?
- @plugin_path = File.expand_path File.join(*args)
- load_map
- end
- @plugin_path
- end
-
- # The host's ID.
- #
- # If PLUGIN_HOST_ID is not set, it is simply the class name.
- def host_id
- if self.const_defined? :PLUGIN_HOST_ID
- self::PLUGIN_HOST_ID
- else
- name
- end
- end
-
- # Map a plugin_id to another.
- #
- # Usage: Put this in a file plugin_path/_map.rb.
- #
- # class MyColorHost < PluginHost
- # map :navy => :dark_blue,
- # :maroon => :brown,
- # :luna => :moon
- # end
- def map hash
- for from, to in hash
- from = validate_id from
- to = validate_id to
- plugin_hash[from] = to unless plugin_hash.has_key? from
- end
- end
-
- # Define the default plugin to use when no plugin is found
- # for a given id.
- #
- # See also map.
- #
- # class MyColorHost < PluginHost
- # map :navy => :dark_blue
- # default :gray
- # end
- def default id = nil
- if id
- id = validate_id id
- plugin_hash[nil] = id
- else
- plugin_hash[nil]
- end
- end
-
- # Every plugin must register itself for one or more
- # +ids+ by calling register_for, which calls this method.
- #
- # See Plugin#register_for.
- def register plugin, *ids
- for id in ids
- unless id.is_a? Symbol
- raise ArgumentError,
- "id must be a Symbol, but it was a #{id.class}"
- end
- plugin_hash[validate_id(id)] = plugin
- end
- end
-
- # A Hash of plugion_id => Plugin pairs.
- def plugin_hash
- @plugin_hash ||= create_plugin_hash
- end
-
- # Returns an array of all .rb files in the plugin path.
- #
- # The extension .rb is not included.
- def list
- Dir[path_to('*')].select do |file|
- File.basename(file)[/^(?!_)\w+\.rb$/]
- end.map do |file|
- File.basename file, '.rb'
- end
- end
-
- # Makes a map of all loaded plugins.
- def inspect
- map = plugin_hash.dup
- map.each do |id, plugin|
- map[id] = plugin.to_s[/(?>\w+)$/]
- end
- "#{name}[#{host_id}]#{map.inspect}"
- end
-
-protected
- # Created a new plugin list and stores it to @plugin_hash.
- def create_plugin_hash
- @plugin_hash =
- Hash.new do |h, plugin_id|
- id = validate_id(plugin_id)
- path = path_to id
- begin
- require path
- rescue LoadError => boom
- if h.has_key? nil # default plugin
- h[id] = h[nil]
- else
- raise PluginNotFound, 'Could not load plugin %p: %s' % [id, boom]
- end
- else
- # Plugin should have registered by now
- unless h.has_key? id
- raise PluginNotFound,
- "No #{self.name} plugin for #{id.inspect} found in #{path}."
- end
- end
- h[id]
- end
- end
-
- # Loads the map file (see map).
- #
- # This is done automatically when plugin_path is called.
- def load_map
- mapfile = path_to '_map'
- if File.exist? mapfile
- require mapfile
- elsif $VERBOSE
- warn 'no _map.rb found for %s' % name
- end
- end
-
- # Returns the Plugin for +id+.
- # Use it like Hash#fetch.
- #
- # Example:
- # yaml_plugin = MyPluginHost[:yaml, :default]
- def fetch id, *args, &blk
- plugin_hash.fetch validate_id(id), *args, &blk
- end
-
- # Returns the expected path to the plugin file for the given id.
- def path_to plugin_id
- File.join plugin_path, "#{plugin_id}.rb"
- end
-
- # Converts +id+ to a Symbol if it is a String,
- # or returns +id+ if it already is a Symbol.
- #
- # Raises +ArgumentError+ for all other objects, or if the
- # given String includes non-alphanumeric characters (\W).
- def validate_id id
- if id.is_a? Symbol or id.nil?
- id
- elsif id.is_a? String
- if id[/\w+/] == id
- id.downcase.to_sym
- else
- raise ArgumentError, "Invalid id: '#{id}' given."
- end
- else
- raise ArgumentError,
- "String or Symbol expected, but #{id.class} given."
- end
- end
-
-end
-
-
-# = Plugin
-#
-# Plugins have to include this module.
-#
-# IMPORTANT: use extend for this module.
-#
-# Example: see PluginHost.
-module Plugin
-
- def included mod
- warn "#{name} should not be included. Use extend."
- end
-
- # Register this class for the given langs.
- # Example:
- # class MyPlugin < PluginHost::BaseClass
- # register_for :my_id
- # ...
- # end
- #
- # See PluginHost.register.
- def register_for *ids
- plugin_host.register self, *ids
- end
-
- # Returns the title of the plugin, or sets it to the
- # optional argument +title+.
- def title title = nil
- if title
- @title = title.to_s
- else
- @title ||= name[/([^:]+)$/, 1]
- end
- end
-
- # The host for this Plugin class.
- def plugin_host host = nil
- if host and not host.is_a? PluginHost
- raise ArgumentError,
- "PluginHost expected, but #{host.class} given."
- end
- self.const_set :PLUGIN_HOST, host if host
- self::PLUGIN_HOST
- end
-
- # Require some helper files.
- #
- # Example:
- #
- # class MyPlugin < PluginHost::BaseClass
- # register_for :my_id
- # helper :my_helper
- #
- # The above example loads the file myplugin/my_helper.rb relative to the
- # file in which MyPlugin was defined.
- #
- # You can also load a helper from a different plugin:
- #
- # helper 'other_plugin/helper_name'
- def helper *helpers
- for helper in helpers
- if helper.is_a?(String) && helper[/\//]
- self::PLUGIN_HOST.require_helper $`, $'
- else
- self::PLUGIN_HOST.require_helper plugin_id, helper.to_s
- end
- end
- end
-
- # Returns the pulgin id used by the engine.
- def plugin_id
- name[/\w+$/].downcase
- end
-
-end
-
-# Convenience method for plugin loading.
-# The syntax used is:
-#
-# CodeRay.require_plugin '<Host ID>/<Plugin ID>'
-#
-# Returns the loaded plugin.
-def self.require_plugin path
- host_id, plugin_id = path.split '/', 2
- host = PluginHost.host_by_id(host_id)
- raise PluginHost::HostNotFound,
- "No host for #{host_id.inspect} found." unless host
- host.load plugin_id
-end
-
-end
\ No newline at end of file
+++ /dev/null
-module CodeRay
-
-# = WordList
-#
-# <b>A Hash subclass designed for mapping word lists to token types.</b>
-#
-# Copyright (c) 2006 by murphy (Kornelius Kalnbach) <murphy rubychan de>
-#
-# License:: LGPL / ask the author
-# Version:: 1.1 (2006-Oct-19)
-#
-# A WordList is a Hash with some additional features.
-# It is intended to be used for keyword recognition.
-#
-# WordList is highly optimized to be used in Scanners,
-# typically to decide whether a given ident is a special token.
-#
-# For case insensitive words use CaseIgnoringWordList.
-#
-# Example:
-#
-# # define word arrays
-# RESERVED_WORDS = %w[
-# asm break case continue default do else
-# ...
-# ]
-#
-# PREDEFINED_TYPES = %w[
-# int long short char void
-# ...
-# ]
-#
-# PREDEFINED_CONSTANTS = %w[
-# EOF NULL ...
-# ]
-#
-# # make a WordList
-# IDENT_KIND = WordList.new(:ident).
-# add(RESERVED_WORDS, :reserved).
-# add(PREDEFINED_TYPES, :pre_type).
-# add(PREDEFINED_CONSTANTS, :pre_constant)
-#
-# ...
-#
-# def scan_tokens tokens, options
-# ...
-#
-# elsif scan(/[A-Za-z_][A-Za-z_0-9]*/)
-# # use it
-# kind = IDENT_KIND[match]
-# ...
-class WordList < Hash
-
- # Creates a new WordList with +default+ as default value.
- #
- # You can activate +caching+ to store the results for every [] request.
- #
- # With caching, methods like +include?+ or +delete+ may no longer behave
- # as you expect. Therefore, it is recommended to use the [] method only.
- def initialize default = false, caching = false, &block
- if block
- raise ArgumentError, 'Can\'t combine block with caching.' if caching
- super(&block)
- else
- if caching
- super() do |h, k|
- h[k] = h.fetch k, default
- end
- else
- super default
- end
- end
- end
-
- # Add words to the list and associate them with +kind+.
- #
- # Returns +self+, so you can concat add calls.
- def add words, kind = true
- words.each do |word|
- self[word] = kind
- end
- self
- end
-
-end
-
-
-# A CaseIgnoringWordList is like a WordList, only that
-# keys are compared case-insensitively.
-#
-# Ignoring the text case is realized by sending the +downcase+ message to
-# all keys.
-#
-# Caching usually makes a CaseIgnoringWordList faster, but it has to be
-# activated explicitely.
-class CaseIgnoringWordList < WordList
-
- # Creates a new case-insensitive WordList with +default+ as default value.
- #
- # You can activate caching to store the results for every [] request.
- # This speeds up subsequent lookups for the same word, but also
- # uses memory.
- def initialize default = false, caching = false
- if caching
- super(default, false) do |h, k|
- h[k] = h.fetch k.downcase, default
- end
- else
- super(default, false)
- extend Uncached
- end
- end
-
- module Uncached # :nodoc:
- def [] key
- super(key.downcase)
- end
- end
-
- # Add +words+ to the list and associate them with +kind+.
- def add words, kind = true
- words.each do |word|
- self[word.downcase] = kind
- end
- self
- end
-
-end
-
-end
-
-__END__
-# check memory consumption
-END {
- ObjectSpace.each_object(CodeRay::CaseIgnoringWordList) do |wl|
- p wl.inject(0) { |memo, key, value| memo + key.size + 24 }
- end
-}
\ No newline at end of file
+++ /dev/null
-module CodeRay
-
- require 'coderay/helpers/plugin'
-
- # = Scanners
- #
- # This module holds the Scanner class and its subclasses.
- # For example, the Ruby scanner is named CodeRay::Scanners::Ruby
- # can be found in coderay/scanners/ruby.
- #
- # Scanner also provides methods and constants for the register
- # mechanism and the [] method that returns the Scanner class
- # belonging to the given lang.
- #
- # See PluginHost.
- module Scanners
- extend PluginHost
- plugin_path File.dirname(__FILE__), 'scanners'
-
- require 'strscan'
-
- # = Scanner
- #
- # The base class for all Scanners.
- #
- # It is a subclass of Ruby's great +StringScanner+, which
- # makes it easy to access the scanning methods inside.
- #
- # It is also +Enumerable+, so you can use it like an Array of
- # Tokens:
- #
- # require 'coderay'
- #
- # c_scanner = CodeRay::Scanners[:c].new "if (*p == '{') nest++;"
- #
- # for text, kind in c_scanner
- # puts text if kind == :operator
- # end
- #
- # # prints: (*==)++;
- #
- # OK, this is a very simple example :)
- # You can also use +map+, +any?+, +find+ and even +sort_by+,
- # if you want.
- class Scanner < StringScanner
-
- extend Plugin
- plugin_host Scanners
-
- # Raised if a Scanner fails while scanning
- ScanError = Class.new(Exception)
-
- require 'coderay/helpers/word_list'
-
- # The default options for all scanner classes.
- #
- # Define @default_options for subclasses.
- DEFAULT_OPTIONS = { :stream => false }
-
- KINDS_NOT_LOC = [:comment, :doctype]
-
- class << self
-
- # Returns if the Scanner can be used in streaming mode.
- def streamable?
- is_a? Streamable
- end
-
- def normify code
- code = code.to_s
- if code.respond_to? :force_encoding
- debug, $DEBUG = $DEBUG, false
- begin
- code.force_encoding 'utf-8'
- code[/\z/] # raises an ArgumentError when code contains a non-UTF-8 char
- rescue ArgumentError
- code.force_encoding 'binary'
- ensure
- $DEBUG = debug
- end
- end
- code.to_unix
- end
-
- def file_extension extension = nil
- if extension
- @file_extension = extension.to_s
- else
- @file_extension ||= plugin_id.to_s
- end
- end
-
- end
-
-=begin
-## Excluded for speed reasons; protected seems to make methods slow.
-
- # Save the StringScanner methods from being called.
- # This would not be useful for highlighting.
- strscan_public_methods =
- StringScanner.instance_methods -
- StringScanner.ancestors[1].instance_methods
- protected(*strscan_public_methods)
-=end
-
- # Create a new Scanner.
- #
- # * +code+ is the input String and is handled by the superclass
- # StringScanner.
- # * +options+ is a Hash with Symbols as keys.
- # It is merged with the default options of the class (you can
- # overwrite default options here.)
- # * +block+ is the callback for streamed highlighting.
- #
- # If you set :stream to +true+ in the options, the Scanner uses a
- # TokenStream with the +block+ as callback to handle the tokens.
- #
- # Else, a Tokens object is used.
- def initialize code='', options = {}, &block
- raise "I am only the basic Scanner class. I can't scan "\
- "anything. :( Use my subclasses." if self.class == Scanner
-
- @options = self.class::DEFAULT_OPTIONS.merge options
-
- super Scanner.normify(code)
-
- @tokens = options[:tokens]
- if @options[:stream]
- warn "warning in CodeRay::Scanner.new: :stream is set, "\
- "but no block was given" unless block_given?
- raise NotStreamableError, self unless kind_of? Streamable
- @tokens ||= TokenStream.new(&block)
- else
- warn "warning in CodeRay::Scanner.new: Block given, "\
- "but :stream is #{@options[:stream]}" if block_given?
- @tokens ||= Tokens.new
- end
- @tokens.scanner = self
-
- setup
- end
-
- def reset
- super
- reset_instance
- end
-
- def string= code
- code = Scanner.normify(code)
- super code
- reset_instance
- end
-
- # More mnemonic accessor name for the input string.
- alias code string
- alias code= string=
-
- # Returns the Plugin ID for this scanner.
- def lang
- self.class.plugin_id
- end
-
- # Scans the code and returns all tokens in a Tokens object.
- def tokenize new_string=nil, options = {}
- options = @options.merge(options)
- self.string = new_string if new_string
- @cached_tokens =
- if @options[:stream] # :stream must have been set already
- reset unless new_string
- scan_tokens @tokens, options
- @tokens
- else
- scan_tokens @tokens, options
- end
- end
-
- def tokens
- @cached_tokens ||= tokenize
- end
-
- # Whether the scanner is in streaming mode.
- def streaming?
- !!@options[:stream]
- end
-
- # Traverses the tokens.
- def each &block
- raise ArgumentError,
- 'Cannot traverse TokenStream.' if @options[:stream]
- tokens.each(&block)
- end
- include Enumerable
-
- # The current line position of the scanner.
- #
- # Beware, this is implemented inefficiently. It should be used
- # for debugging only.
- def line
- string[0..pos].count("\n") + 1
- end
-
- def column pos = self.pos
- return 0 if pos <= 0
- string = string()
- if string.respond_to?(:bytesize) && (defined?(@bin_string) || string.bytesize != string.size)
- @bin_string ||= string.dup.force_encoding('binary')
- string = @bin_string
- end
- pos - (string.rindex(?\n, pos) || 0)
- end
-
- def marshal_dump
- @options
- end
-
- def marshal_load options
- @options = options
- end
-
- protected
-
- # Can be implemented by subclasses to do some initialization
- # that has to be done once per instance.
- #
- # Use reset for initialization that has to be done once per
- # scan.
- def setup
- end
-
- # This is the central method, and commonly the only one a
- # subclass implements.
- #
- # Subclasses must implement this method; it must return +tokens+
- # and must only use Tokens#<< for storing scanned tokens!
- def scan_tokens tokens, options
- raise NotImplementedError,
- "#{self.class}#scan_tokens not implemented."
- end
-
- def reset_instance
- @tokens.clear unless @options[:keep_tokens]
- @cached_tokens = nil
- @bin_string = nil if defined? @bin_string
- end
-
- # Scanner error with additional status information
- def raise_inspect msg, tokens, state = 'No state given!', ambit = 30
- raise ScanError, <<-EOE % [
-
-
-***ERROR in %s: %s (after %d tokens)
-
-tokens:
-%s
-
-current line: %d column: %d pos: %d
-matched: %p state: %p
-bol? = %p, eos? = %p
-
-surrounding code:
-%p ~~ %p
-
-
-***ERROR***
-
- EOE
- File.basename(caller[0]),
- msg,
- tokens.size,
- tokens.last(10).map { |t| t.inspect }.join("\n"),
- line, column, pos,
- matched, state, bol?, eos?,
- string[pos - ambit, ambit],
- string[pos, ambit],
- ]
- end
-
- end
-
- end
-end
-
-class String
- # I love this hack. It seems to silence all dos/unix/mac newline problems.
- def to_unix
- if index ?\r
- gsub(/\r\n?/, "\n")
- else
- self
- end
- end
-end
+++ /dev/null
-module CodeRay
-module Scanners
-
- map \
- :h => :c,
- :cplusplus => :cpp,
- :'c++' => :cpp,
- :ecma => :java_script,
- :ecmascript => :java_script,
- :ecma_script => :java_script,
- :irb => :ruby,
- :javascript => :java_script,
- :js => :java_script,
- :nitro => :nitro_xhtml,
- :pascal => :delphi,
- :plain => :plaintext,
- :xhtml => :html,
- :yml => :yaml
-
- default :plain
-
-end
-end
+++ /dev/null
-module CodeRay
-module Scanners
-
- class C < Scanner
-
- include Streamable
-
- register_for :c
- file_extension 'c'
-
- RESERVED_WORDS = [
- 'asm', 'break', 'case', 'continue', 'default', 'do',
- 'else', 'enum', 'for', 'goto', 'if', 'return',
- 'sizeof', 'struct', 'switch', 'typedef', 'union', 'while',
- 'restrict', # added in C99
- ]
-
- PREDEFINED_TYPES = [
- 'int', 'long', 'short', 'char',
- 'signed', 'unsigned', 'float', 'double',
- 'bool', 'complex', # added in C99
- ]
-
- PREDEFINED_CONSTANTS = [
- 'EOF', 'NULL',
- 'true', 'false', # added in C99
- ]
- DIRECTIVES = [
- 'auto', 'extern', 'register', 'static', 'void',
- 'const', 'volatile', # added in C89
- 'inline', # added in C99
- ]
-
- IDENT_KIND = WordList.new(:ident).
- add(RESERVED_WORDS, :reserved).
- add(PREDEFINED_TYPES, :pre_type).
- add(DIRECTIVES, :directive).
- add(PREDEFINED_CONSTANTS, :pre_constant)
-
- ESCAPE = / [rbfntv\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} /x
- UNICODE_ESCAPE = / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} /x
-
- def scan_tokens tokens, options
-
- state = :initial
- label_expected = true
- case_expected = false
- label_expected_before_preproc_line = nil
- in_preproc_line = false
-
- until eos?
-
- kind = nil
- match = nil
-
- case state
-
- when :initial
-
- if match = scan(/ \s+ | \\\n /x)
- if in_preproc_line && match != "\\\n" && match.index(?\n)
- in_preproc_line = false
- label_expected = label_expected_before_preproc_line
- end
- tokens << [match, :space]
- next
-
- elsif scan(%r! // [^\n\\]* (?: \\. [^\n\\]* )* | /\* (?: .*? \*/ | .* ) !mx)
- kind = :comment
-
- elsif match = scan(/ \# \s* if \s* 0 /x)
- match << scan_until(/ ^\# (?:elif|else|endif) .*? $ | \z /xm) unless eos?
- kind = :comment
-
- elsif match = scan(/ [-+*=<>?:;,!&^|()\[\]{}~%]+ | \/=? | \.(?!\d) /x)
- label_expected = match =~ /[;\{\}]/
- if case_expected
- label_expected = true if match == ':'
- case_expected = false
- end
- kind = :operator
-
- elsif match = scan(/ [A-Za-z_][A-Za-z_0-9]* /x)
- kind = IDENT_KIND[match]
- if kind == :ident && label_expected && !in_preproc_line && scan(/:(?!:)/)
- kind = :label
- match << matched
- else
- label_expected = false
- if kind == :reserved
- case match
- when 'case', 'default'
- case_expected = true
- end
- end
- end
-
- elsif scan(/\$/)
- kind = :ident
-
- elsif match = scan(/L?"/)
- tokens << [:open, :string]
- if match[0] == ?L
- tokens << ['L', :modifier]
- match = '"'
- end
- state = :string
- kind = :delimiter
-
- elsif scan(/#[ \t]*(\w*)/)
- kind = :preprocessor
- in_preproc_line = true
- label_expected_before_preproc_line = label_expected
- state = :include_expected if self[1] == 'include'
-
- elsif scan(/ L?' (?: [^\'\n\\] | \\ #{ESCAPE} )? '? /ox)
- label_expected = false
- kind = :char
-
- elsif scan(/0[xX][0-9A-Fa-f]+/)
- label_expected = false
- kind = :hex
-
- elsif scan(/(?:0[0-7]+)(?![89.eEfF])/)
- label_expected = false
- kind = :oct
-
- elsif scan(/(?:\d+)(?![.eEfF])L?L?/)
- label_expected = false
- kind = :integer
-
- elsif scan(/\d[fF]?|\d*\.\d+(?:[eE][+-]?\d+)?[fF]?|\d+[eE][+-]?\d+[fF]?/)
- label_expected = false
- kind = :float
-
- else
- getch
- kind = :error
-
- end
-
- when :string
- if scan(/[^\\\n"]+/)
- kind = :content
- elsif scan(/"/)
- tokens << ['"', :delimiter]
- tokens << [:close, :string]
- state = :initial
- label_expected = false
- next
- elsif scan(/ \\ (?: #{ESCAPE} | #{UNICODE_ESCAPE} ) /mox)
- kind = :char
- elsif scan(/ \\ | $ /x)
- tokens << [:close, :string]
- kind = :error
- state = :initial
- label_expected = false
- else
- raise_inspect "else case \" reached; %p not handled." % peek(1), tokens
- end
-
- when :include_expected
- if scan(/<[^>\n]+>?|"[^"\n\\]*(?:\\.[^"\n\\]*)*"?/)
- kind = :include
- state = :initial
-
- elsif match = scan(/\s+/)
- kind = :space
- state = :initial if match.index ?\n
-
- else
- state = :initial
- next
-
- end
-
- else
- raise_inspect 'Unknown state', tokens
-
- end
-
- match ||= matched
- if $CODERAY_DEBUG and not kind
- raise_inspect 'Error token %p in line %d' %
- [[match, kind], line], tokens
- end
- raise_inspect 'Empty token', tokens unless match
-
- tokens << [match, kind]
-
- end
-
- if state == :string
- tokens << [:close, :string]
- end
-
- tokens
- end
-
- end
-
-end
-end
+++ /dev/null
-module CodeRay
-module Scanners
-
- class CPlusPlus < Scanner
-
- include Streamable
-
- register_for :cpp
- file_extension 'cpp'
- title 'C++'
-
- # http://www.cppreference.com/wiki/keywords/start
- RESERVED_WORDS = [
- 'and', 'and_eq', 'asm', 'bitand', 'bitor', 'break',
- 'case', 'catch', 'class', 'compl', 'const_cast',
- 'continue', 'default', 'delete', 'do', 'dynamic_cast', 'else',
- 'enum', 'export', 'for', 'goto', 'if', 'namespace', 'new',
- 'not', 'not_eq', 'or', 'or_eq', 'reinterpret_cast', 'return',
- 'sizeof', 'static_cast', 'struct', 'switch', 'template',
- 'throw', 'try', 'typedef', 'typeid', 'typename', 'union',
- 'while', 'xor', 'xor_eq'
- ]
-
- PREDEFINED_TYPES = [
- 'bool', 'char', 'double', 'float', 'int', 'long',
- 'short', 'signed', 'unsigned', 'wchar_t', 'string'
- ]
- PREDEFINED_CONSTANTS = [
- 'false', 'true',
- 'EOF', 'NULL',
- ]
- PREDEFINED_VARIABLES = [
- 'this'
- ]
- DIRECTIVES = [
- 'auto', 'const', 'explicit', 'extern', 'friend', 'inline', 'mutable', 'operator',
- 'private', 'protected', 'public', 'register', 'static', 'using', 'virtual', 'void',
- 'volatile'
- ]
-
- IDENT_KIND = WordList.new(:ident).
- add(RESERVED_WORDS, :reserved).
- add(PREDEFINED_TYPES, :pre_type).
- add(PREDEFINED_VARIABLES, :local_variable).
- add(DIRECTIVES, :directive).
- add(PREDEFINED_CONSTANTS, :pre_constant)
-
- ESCAPE = / [rbfntv\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} /x
- UNICODE_ESCAPE = / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} /x
-
- def scan_tokens tokens, options
-
- state = :initial
- label_expected = true
- case_expected = false
- label_expected_before_preproc_line = nil
- in_preproc_line = false
-
- until eos?
-
- kind = nil
- match = nil
-
- case state
-
- when :initial
-
- if match = scan(/ \s+ | \\\n /x)
- if in_preproc_line && match != "\\\n" && match.index(?\n)
- in_preproc_line = false
- label_expected = label_expected_before_preproc_line
- end
- tokens << [match, :space]
- next
-
- elsif scan(%r! // [^\n\\]* (?: \\. [^\n\\]* )* | /\* (?: .*? \*/ | .* ) !mx)
- kind = :comment
-
- elsif match = scan(/ \# \s* if \s* 0 /x)
- match << scan_until(/ ^\# (?:elif|else|endif) .*? $ | \z /xm) unless eos?
- kind = :comment
-
- elsif match = scan(/ [-+*=<>?:;,!&^|()\[\]{}~%]+ | \/=? | \.(?!\d) /x)
- label_expected = match =~ /[;\{\}]/
- if case_expected
- label_expected = true if match == ':'
- case_expected = false
- end
- kind = :operator
-
- elsif match = scan(/ [A-Za-z_][A-Za-z_0-9]* /x)
- kind = IDENT_KIND[match]
- if kind == :ident && label_expected && !in_preproc_line && scan(/:(?!:)/)
- kind = :label
- match << matched
- else
- label_expected = false
- if kind == :reserved
- case match
- when 'class'
- state = :class_name_expected
- when 'case', 'default'
- case_expected = true
- end
- end
- end
-
- elsif scan(/\$/)
- kind = :ident
-
- elsif match = scan(/L?"/)
- tokens << [:open, :string]
- if match[0] == ?L
- tokens << ['L', :modifier]
- match = '"'
- end
- state = :string
- kind = :delimiter
-
- elsif scan(/#[ \t]*(\w*)/)
- kind = :preprocessor
- in_preproc_line = true
- label_expected_before_preproc_line = label_expected
- state = :include_expected if self[1] == 'include'
-
- elsif scan(/ L?' (?: [^\'\n\\] | \\ #{ESCAPE} )? '? /ox)
- label_expected = false
- kind = :char
-
- elsif scan(/0[xX][0-9A-Fa-f]+/)
- label_expected = false
- kind = :hex
-
- elsif scan(/(?:0[0-7]+)(?![89.eEfF])/)
- label_expected = false
- kind = :oct
-
- elsif scan(/(?:\d+)(?![.eEfF])L?L?/)
- label_expected = false
- kind = :integer
-
- elsif scan(/\d[fF]?|\d*\.\d+(?:[eE][+-]?\d+)?[fF]?|\d+[eE][+-]?\d+[fF]?/)
- label_expected = false
- kind = :float
-
- else
- getch
- kind = :error
-
- end
-
- when :string
- if scan(/[^\\"]+/)
- kind = :content
- elsif scan(/"/)
- tokens << ['"', :delimiter]
- tokens << [:close, :string]
- state = :initial
- label_expected = false
- next
- elsif scan(/ \\ (?: #{ESCAPE} | #{UNICODE_ESCAPE} ) /mox)
- kind = :char
- elsif scan(/ \\ | $ /x)
- tokens << [:close, :string]
- kind = :error
- state = :initial
- label_expected = false
- else
- raise_inspect "else case \" reached; %p not handled." % peek(1), tokens
- end
-
- when :include_expected
- if scan(/<[^>\n]+>?|"[^"\n\\]*(?:\\.[^"\n\\]*)*"?/)
- kind = :include
- state = :initial
-
- elsif match = scan(/\s+/)
- kind = :space
- state = :initial if match.index ?\n
-
- else
- state = :initial
- next
-
- end
-
- when :class_name_expected
- if scan(/ [A-Za-z_][A-Za-z_0-9]* /x)
- kind = :class
- state = :initial
-
- elsif match = scan(/\s+/)
- kind = :space
-
- else
- getch
- kind = :error
- state = :initial
-
- end
-
- else
- raise_inspect 'Unknown state', tokens
-
- end
-
- match ||= matched
- if $CODERAY_DEBUG and not kind
- raise_inspect 'Error token %p in line %d' %
- [[match, kind], line], tokens
- end
- raise_inspect 'Empty token', tokens unless match
-
- tokens << [match, kind]
-
- end
-
- if state == :string
- tokens << [:close, :string]
- end
-
- tokens
- end
-
- end
-
-end
-end
+++ /dev/null
-module CodeRay
-module Scanners
-
- class CSS < Scanner
-
- register_for :css
-
- KINDS_NOT_LOC = [
- :comment,
- :class, :pseudo_class, :type,
- :constant, :directive,
- :key, :value, :operator, :color, :float,
- :error, :important,
- ]
-
- module RE
- NonASCII = /[\x80-\xFF]/
- Hex = /[0-9a-fA-F]/
- Unicode = /\\#{Hex}{1,6}(?:\r\n|\s)?/ # differs from standard because it allows uppercase hex too
- Escape = /#{Unicode}|\\[^\r\n\f0-9a-fA-F]/
- NMChar = /[-_a-zA-Z0-9]|#{NonASCII}|#{Escape}/
- NMStart = /[_a-zA-Z]|#{NonASCII}|#{Escape}/
- NL = /\r\n|\r|\n|\f/
- String1 = /"(?:[^\n\r\f\\"]|\\#{NL}|#{Escape})*"?/ # FIXME: buggy regexp
- String2 = /'(?:[^\n\r\f\\']|\\#{NL}|#{Escape})*'?/ # FIXME: buggy regexp
- String = /#{String1}|#{String2}/
-
- HexColor = /#(?:#{Hex}{6}|#{Hex}{3})/
- Color = /#{HexColor}/
-
- Num = /-?(?:[0-9]+|[0-9]*\.[0-9]+)/
- Name = /#{NMChar}+/
- Ident = /-?#{NMStart}#{NMChar}*/
- AtKeyword = /@#{Ident}/
- Percentage = /#{Num}%/
-
- reldimensions = %w[em ex px]
- absdimensions = %w[in cm mm pt pc]
- Unit = Regexp.union(*(reldimensions + absdimensions))
-
- Dimension = /#{Num}#{Unit}/
-
- Comment = %r! /\* (?: .*? \*/ | .* ) !mx
- Function = /(?:url|alpha)\((?:[^)\n\r\f]|\\\))*\)?/
-
- Id = /##{Name}/
- Class = /\.#{Name}/
- PseudoClass = /:#{Name}/
- AttributeSelector = /\[[^\]]*\]?/
-
- end
-
- def scan_tokens tokens, options
-
- value_expected = nil
- states = [:initial]
-
- until eos?
-
- kind = nil
- match = nil
-
- if scan(/\s+/)
- kind = :space
-
- elsif case states.last
- when :initial, :media
- if scan(/(?>#{RE::Ident})(?!\()|\*/ox)
- kind = :type
- elsif scan RE::Class
- kind = :class
- elsif scan RE::Id
- kind = :constant
- elsif scan RE::PseudoClass
- kind = :pseudo_class
- elsif match = scan(RE::AttributeSelector)
- # TODO: Improve highlighting inside of attribute selectors.
- tokens << [:open, :string]
- tokens << [match[0,1], :delimiter]
- tokens << [match[1..-2], :content] if match.size > 2
- tokens << [match[-1,1], :delimiter] if match[-1] == ?]
- tokens << [:close, :string]
- next
- elsif match = scan(/@media/)
- kind = :directive
- states.push :media_before_name
- end
-
- when :block
- if scan(/(?>#{RE::Ident})(?!\()/ox)
- if value_expected
- kind = :value
- else
- kind = :key
- end
- end
-
- when :media_before_name
- if scan RE::Ident
- kind = :type
- states[-1] = :media_after_name
- end
-
- when :media_after_name
- if scan(/\{/)
- kind = :operator
- states[-1] = :media
- end
-
- when :comment
- if scan(/(?:[^*\s]|\*(?!\/))+/)
- kind = :comment
- elsif scan(/\*\//)
- kind = :comment
- states.pop
- elsif scan(/\s+/)
- kind = :space
- end
-
- else
- raise_inspect 'Unknown state', tokens
-
- end
-
- elsif scan(/\/\*/)
- kind = :comment
- states.push :comment
-
- elsif scan(/\{/)
- value_expected = false
- kind = :operator
- states.push :block
-
- elsif scan(/\}/)
- value_expected = false
- if states.last == :block || states.last == :media
- kind = :operator
- states.pop
- else
- kind = :error
- end
-
- elsif match = scan(/#{RE::String}/o)
- tokens << [:open, :string]
- tokens << [match[0, 1], :delimiter]
- tokens << [match[1..-2], :content] if match.size > 2
- tokens << [match[-1, 1], :delimiter] if match.size >= 2
- tokens << [:close, :string]
- next
-
- elsif match = scan(/#{RE::Function}/o)
- tokens << [:open, :string]
- start = match[/^\w+\(/]
- tokens << [start, :delimiter]
- if match[-1] == ?)
- tokens << [match[start.size..-2], :content]
- tokens << [')', :delimiter]
- else
- tokens << [match[start.size..-1], :content]
- end
- tokens << [:close, :string]
- next
-
- elsif scan(/(?: #{RE::Dimension} | #{RE::Percentage} | #{RE::Num} )/ox)
- kind = :float
-
- elsif scan(/#{RE::Color}/o)
- kind = :color
-
- elsif scan(/! *important/)
- kind = :important
-
- elsif scan(/rgb\([^()\n]*\)?/)
- kind = :color
-
- elsif scan(/#{RE::AtKeyword}/o)
- kind = :directive
-
- elsif match = scan(/ [+>:;,.=()\/] /x)
- if match == ':'
- value_expected = true
- elsif match == ';'
- value_expected = false
- end
- kind = :operator
-
- else
- getch
- kind = :error
-
- end
-
- match ||= matched
- if $CODERAY_DEBUG and not kind
- raise_inspect 'Error token %p in line %d' %
- [[match, kind], line], tokens
- end
- raise_inspect 'Empty token', tokens unless match
-
- tokens << [match, kind]
-
- end
-
- tokens
- end
-
- end
-
-end
-end
+++ /dev/null
-module CodeRay
-module Scanners
-
- # = Debug Scanner
- class Debug < Scanner
-
- include Streamable
- register_for :debug
- file_extension 'raydebug'
- title 'CodeRay Token Dump'
-
- protected
- def scan_tokens tokens, options
-
- opened_tokens = []
-
- until eos?
-
- kind = nil
- match = nil
-
- if scan(/\s+/)
- tokens << [matched, :space]
- next
-
- elsif scan(/ (\w+) \( ( [^\)\\]* ( \\. [^\)\\]* )* ) \) /x)
- kind = self[1].to_sym
- match = self[2].gsub(/\\(.)/, '\1')
-
- elsif scan(/ (\w+) < /x)
- kind = self[1].to_sym
- opened_tokens << kind
- match = :open
-
- elsif !opened_tokens.empty? && scan(/ > /x)
- kind = opened_tokens.pop || :error
- match = :close
-
- else
- kind = :error
- getch
-
- end
-
- match ||= matched
- if $CODERAY_DEBUG and not kind
- raise_inspect 'Error token %p in line %d' %
- [[match, kind], line], tokens
- end
- raise_inspect 'Empty token', tokens unless match
-
- tokens << [match, kind]
-
- end
-
- tokens
- end
-
- end
-
-end
-end
+++ /dev/null
-module CodeRay
-module Scanners
-
- class Delphi < Scanner
-
- register_for :delphi
- file_extension 'pas'
-
- RESERVED_WORDS = [
- 'and', 'array', 'as', 'at', 'asm', 'at', 'begin', 'case', 'class',
- 'const', 'constructor', 'destructor', 'dispinterface', 'div', 'do',
- 'downto', 'else', 'end', 'except', 'exports', 'file', 'finalization',
- 'finally', 'for', 'function', 'goto', 'if', 'implementation', 'in',
- 'inherited', 'initialization', 'inline', 'interface', 'is', 'label',
- 'library', 'mod', 'nil', 'not', 'object', 'of', 'or', 'out', 'packed',
- 'procedure', 'program', 'property', 'raise', 'record', 'repeat',
- 'resourcestring', 'set', 'shl', 'shr', 'string', 'then', 'threadvar',
- 'to', 'try', 'type', 'unit', 'until', 'uses', 'var', 'while', 'with',
- 'xor', 'on'
- ]
-
- DIRECTIVES = [
- 'absolute', 'abstract', 'assembler', 'at', 'automated', 'cdecl',
- 'contains', 'deprecated', 'dispid', 'dynamic', 'export',
- 'external', 'far', 'forward', 'implements', 'local',
- 'near', 'nodefault', 'on', 'overload', 'override',
- 'package', 'pascal', 'platform', 'private', 'protected', 'public',
- 'published', 'read', 'readonly', 'register', 'reintroduce',
- 'requires', 'resident', 'safecall', 'stdcall', 'stored', 'varargs',
- 'virtual', 'write', 'writeonly'
- ]
-
- IDENT_KIND = CaseIgnoringWordList.new(:ident).
- add(RESERVED_WORDS, :reserved).
- add(DIRECTIVES, :directive)
-
- NAME_FOLLOWS = CaseIgnoringWordList.new(false).
- add(%w(procedure function .))
-
- private
- def scan_tokens tokens, options
-
- state = :initial
- last_token = ''
-
- until eos?
-
- kind = nil
- match = nil
-
- if state == :initial
-
- if scan(/ \s+ /x)
- tokens << [matched, :space]
- next
-
- elsif scan(%r! \{ \$ [^}]* \}? | \(\* \$ (?: .*? \*\) | .* ) !mx)
- tokens << [matched, :preprocessor]
- next
-
- elsif scan(%r! // [^\n]* | \{ [^}]* \}? | \(\* (?: .*? \*\) | .* ) !mx)
- tokens << [matched, :comment]
- next
-
- elsif match = scan(/ <[>=]? | >=? | :=? | [-+=*\/;,@\^|\(\)\[\]] | \.\. /x)
- kind = :operator
-
- elsif match = scan(/\./)
- kind = :operator
- if last_token == 'end'
- tokens << [match, kind]
- next
- end
-
- elsif match = scan(/ [A-Za-z_][A-Za-z_0-9]* /x)
- kind = NAME_FOLLOWS[last_token] ? :ident : IDENT_KIND[match]
-
- elsif match = scan(/ ' ( [^\n']|'' ) (?:'|$) /x)
- tokens << [:open, :char]
- tokens << ["'", :delimiter]
- tokens << [self[1], :content]
- tokens << ["'", :delimiter]
- tokens << [:close, :char]
- next
-
- elsif match = scan(/ ' /x)
- tokens << [:open, :string]
- state = :string
- kind = :delimiter
-
- elsif scan(/ \# (?: \d+ | \$[0-9A-Fa-f]+ ) /x)
- kind = :char
-
- elsif scan(/ \$ [0-9A-Fa-f]+ /x)
- kind = :hex
-
- elsif scan(/ (?: \d+ ) (?![eE]|\.[^.]) /x)
- kind = :integer
-
- elsif scan(/ \d+ (?: \.\d+ (?: [eE][+-]? \d+ )? | [eE][+-]? \d+ ) /x)
- kind = :float
-
- else
- kind = :error
- getch
-
- end
-
- elsif state == :string
- if scan(/[^\n']+/)
- kind = :content
- elsif scan(/''/)
- kind = :char
- elsif scan(/'/)
- tokens << ["'", :delimiter]
- tokens << [:close, :string]
- state = :initial
- next
- elsif scan(/\n/)
- tokens << [:close, :string]
- kind = :error
- state = :initial
- else
- raise "else case \' reached; %p not handled." % peek(1), tokens
- end
-
- else
- raise 'else-case reached', tokens
-
- end
-
- match ||= matched
- if $CODERAY_DEBUG and not kind
- raise_inspect 'Error token %p in line %d' %
- [[match, kind], line], tokens, state
- end
- raise_inspect 'Empty token', tokens unless match
-
- last_token = match
- tokens << [match, kind]
-
- end
-
- tokens
- end
-
- end
-
-end
-end
+++ /dev/null
-module CodeRay
-module Scanners
-
- class Diff < Scanner
-
- register_for :diff
- title 'diff output'
-
- def scan_tokens tokens, options
-
- line_kind = nil
- state = :initial
-
- until eos?
- kind = match = nil
-
- if match = scan(/\n/)
- if line_kind
- tokens << [:end_line, line_kind]
- line_kind = nil
- end
- tokens << [match, :space]
- next
- end
-
- case state
-
- when :initial
- if match = scan(/--- |\+\+\+ |=+|_+/)
- tokens << [:begin_line, line_kind = :head]
- tokens << [match, :head]
- next unless match = scan(/.+/)
- kind = :plain
- elsif match = scan(/Index: |Property changes on: /)
- tokens << [:begin_line, line_kind = :head]
- tokens << [match, :head]
- next unless match = scan(/.+/)
- kind = :plain
- elsif match = scan(/Added: /)
- tokens << [:begin_line, line_kind = :head]
- tokens << [match, :head]
- next unless match = scan(/.+/)
- kind = :plain
- state = :added
- elsif match = scan(/\\ /)
- tokens << [:begin_line, line_kind = :change]
- tokens << [match, :change]
- next unless match = scan(/.+/)
- kind = :plain
- elsif scan(/(@@)((?>[^@\n]*))(@@)/)
- tokens << [:begin_line, line_kind = :change]
- tokens << [self[1], :change]
- tokens << [self[2], :plain]
- tokens << [self[3], :change]
- next unless match = scan(/.+/)
- kind = :plain
- elsif match = scan(/\+/)
- tokens << [:begin_line, line_kind = :insert]
- tokens << [match, :insert]
- next unless match = scan(/.+/)
- kind = :plain
- elsif match = scan(/-/)
- tokens << [:begin_line, line_kind = :delete]
- tokens << [match, :delete]
- next unless match = scan(/.+/)
- kind = :plain
- elsif scan(/ .*/)
- kind = :comment
- elsif scan(/.+/)
- tokens << [:begin_line, line_kind = :head]
- kind = :plain
- else
- raise_inspect 'else case rached'
- end
-
- when :added
- if match = scan(/ \+/)
- tokens << [:begin_line, line_kind = :insert]
- tokens << [match, :insert]
- next unless match = scan(/.+/)
- kind = :plain
- else
- state = :initial
- next
- end
- end
-
- match ||= matched
- if $CODERAY_DEBUG and not kind
- raise_inspect 'Error token %p in line %d' %
- [[match, kind], line], tokens
- end
- raise_inspect 'Empty token', tokens unless match
-
- tokens << [match, kind]
- end
-
- tokens << [:end_line, line_kind] if line_kind
- tokens
- end
-
- end
-
-end
-end
+++ /dev/null
-module CodeRay
-module Scanners
-
- load :java
-
- class Groovy < Java
-
- include Streamable
- register_for :groovy
-
- # TODO: Check this!
- GROOVY_KEYWORDS = %w[
- as assert def in
- ]
- KEYWORDS_EXPECTING_VALUE = WordList.new.add %w[
- case instanceof new return throw typeof while as assert in
- ]
- GROOVY_MAGIC_VARIABLES = %w[ it ]
-
- IDENT_KIND = Java::IDENT_KIND.dup.
- add(GROOVY_KEYWORDS, :keyword).
- add(GROOVY_MAGIC_VARIABLES, :local_variable)
-
- ESCAPE = / [bfnrtv$\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} /x
- UNICODE_ESCAPE = / u[a-fA-F0-9]{4} /x # no 4-byte unicode chars? U[a-fA-F0-9]{8}
- REGEXP_ESCAPE = / [bfnrtv\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} | \d | [bBdDsSwW\/] /x
-
- # TODO: interpretation inside ', ", /
- STRING_CONTENT_PATTERN = {
- "'" => /(?>\\[^\\'\n]+|[^\\'\n]+)+/,
- '"' => /[^\\$"\n]+/,
- "'''" => /(?>[^\\']+|'(?!''))+/,
- '"""' => /(?>[^\\$"]+|"(?!""))+/,
- '/' => /[^\\$\/\n]+/,
- }
-
- def scan_tokens tokens, options
-
- state = :initial
- inline_block_stack = []
- inline_block_paren_depth = nil
- string_delimiter = nil
- import_clause = class_name_follows = last_token = after_def = false
- value_expected = true
-
- until eos?
-
- kind = nil
- match = nil
-
- case state
-
- when :initial
-
- if match = scan(/ \s+ | \\\n /x)
- tokens << [match, :space]
- if match.index ?\n
- import_clause = after_def = false
- value_expected = true unless value_expected
- end
- next
-
- elsif scan(%r! // [^\n\\]* (?: \\. [^\n\\]* )* | /\* (?: .*? \*/ | .* ) !mx)
- value_expected = true
- after_def = false
- kind = :comment
-
- elsif bol? && scan(/ \#!.* /x)
- kind = :doctype
-
- elsif import_clause && scan(/ (?!as) #{IDENT} (?: \. #{IDENT} )* (?: \.\* )? /ox)
- after_def = value_expected = false
- kind = :include
-
- elsif match = scan(/ #{IDENT} | \[\] /ox)
- kind = IDENT_KIND[match]
- value_expected = (kind == :keyword) && KEYWORDS_EXPECTING_VALUE[match]
- if last_token == '.'
- kind = :ident
- elsif class_name_follows
- kind = :class
- class_name_follows = false
- elsif after_def && check(/\s*[({]/)
- kind = :method
- after_def = false
- elsif kind == :ident && last_token != '?' && check(/:/)
- kind = :key
- else
- class_name_follows = true if match == 'class' || (import_clause && match == 'as')
- import_clause = match == 'import'
- after_def = true if match == 'def'
- end
-
- elsif scan(/;/)
- import_clause = after_def = false
- value_expected = true
- kind = :operator
-
- elsif scan(/\{/)
- class_name_follows = after_def = false
- value_expected = true
- kind = :operator
- if !inline_block_stack.empty?
- inline_block_paren_depth += 1
- end
-
- # TODO: ~'...', ~"..." and ~/.../ style regexps
- elsif match = scan(/ \.\.<? | \*?\.(?!\d)@? | \.& | \?:? | [,?:(\[] | -[->] | \+\+ |
- && | \|\| | \*\*=? | ==?~ | <=?>? | [-+*%^~&|>=!]=? | <<<?=? | >>>?=? /x)
- value_expected = true
- value_expected = :regexp if match == '~'
- after_def = false
- kind = :operator
-
- elsif match = scan(/ [)\]}] /x)
- value_expected = after_def = false
- if !inline_block_stack.empty? && match == '}'
- inline_block_paren_depth -= 1
- if inline_block_paren_depth == 0 # closing brace of inline block reached
- tokens << [match, :inline_delimiter]
- tokens << [:close, :inline]
- state, string_delimiter, inline_block_paren_depth = inline_block_stack.pop
- next
- end
- end
- kind = :operator
-
- elsif check(/[\d.]/)
- after_def = value_expected = false
- if scan(/0[xX][0-9A-Fa-f]+/)
- kind = :hex
- elsif scan(/(?>0[0-7]+)(?![89.eEfF])/)
- kind = :oct
- elsif scan(/\d+[fFdD]|\d*\.\d+(?:[eE][+-]?\d+)?[fFdD]?|\d+[eE][+-]?\d+[fFdD]?/)
- kind = :float
- elsif scan(/\d+[lLgG]?/)
- kind = :integer
- end
-
- elsif match = scan(/'''|"""/)
- after_def = value_expected = false
- state = :multiline_string
- tokens << [:open, :string]
- string_delimiter = match
- kind = :delimiter
-
- # TODO: record.'name'
- elsif match = scan(/["']/)
- after_def = value_expected = false
- state = match == '/' ? :regexp : :string
- tokens << [:open, state]
- string_delimiter = match
- kind = :delimiter
-
- elsif value_expected && (match = scan(/\//))
- after_def = value_expected = false
- tokens << [:open, :regexp]
- state = :regexp
- string_delimiter = '/'
- kind = :delimiter
-
- elsif scan(/ @ #{IDENT} /ox)
- after_def = value_expected = false
- kind = :annotation
-
- elsif scan(/\//)
- after_def = false
- value_expected = true
- kind = :operator
-
- else
- getch
- kind = :error
-
- end
-
- when :string, :regexp, :multiline_string
- if scan(STRING_CONTENT_PATTERN[string_delimiter])
- kind = :content
-
- elsif match = scan(state == :multiline_string ? /'''|"""/ : /["'\/]/)
- tokens << [match, :delimiter]
- if state == :regexp
- # TODO: regexp modifiers? s, m, x, i?
- modifiers = scan(/[ix]+/)
- tokens << [modifiers, :modifier] if modifiers && !modifiers.empty?
- end
- state = :string if state == :multiline_string
- tokens << [:close, state]
- string_delimiter = nil
- after_def = value_expected = false
- state = :initial
- next
-
- elsif (state == :string || state == :multiline_string) &&
- (match = scan(/ \\ (?: #{ESCAPE} | #{UNICODE_ESCAPE} ) /mox))
- if string_delimiter[0] == ?' && !(match == "\\\\" || match == "\\'")
- kind = :content
- else
- kind = :char
- end
- elsif state == :regexp && scan(/ \\ (?: #{REGEXP_ESCAPE} | #{UNICODE_ESCAPE} ) /mox)
- kind = :char
-
- elsif match = scan(/ \$ #{IDENT} /mox)
- tokens << [:open, :inline]
- tokens << ['$', :inline_delimiter]
- match = match[1..-1]
- tokens << [match, IDENT_KIND[match]]
- tokens << [:close, :inline]
- next
- elsif match = scan(/ \$ \{ /x)
- tokens << [:open, :inline]
- tokens << ['${', :inline_delimiter]
- inline_block_stack << [state, string_delimiter, inline_block_paren_depth]
- inline_block_paren_depth = 1
- state = :initial
- next
-
- elsif scan(/ \$ /mx)
- kind = :content
-
- elsif scan(/ \\. /mx)
- kind = :content
-
- elsif scan(/ \\ | \n /x)
- tokens << [:close, state]
- kind = :error
- after_def = value_expected = false
- state = :initial
-
- else
- raise_inspect "else case \" reached; %p not handled." % peek(1), tokens
- end
-
- else
- raise_inspect 'Unknown state', tokens
-
- end
-
- match ||= matched
- if $CODERAY_DEBUG and not kind
- raise_inspect 'Error token %p in line %d' %
- [[match, kind], line], tokens
- end
- raise_inspect 'Empty token', tokens unless match
-
- last_token = match unless [:space, :comment, :doctype].include? kind
-
- tokens << [match, kind]
-
- end
-
- if [:multiline_string, :string, :regexp].include? state
- tokens << [:close, state]
- end
-
- tokens
- end
-
- end
-
-end
-end
+++ /dev/null
-module CodeRay
-module Scanners
-
- # HTML Scanner
- class HTML < Scanner
-
- include Streamable
- register_for :html
-
- KINDS_NOT_LOC = [
- :comment, :doctype, :preprocessor,
- :tag, :attribute_name, :operator,
- :attribute_value, :delimiter, :content,
- :plain, :entity, :error
- ]
-
- ATTR_NAME = /[\w.:-]+/
- ATTR_VALUE_UNQUOTED = ATTR_NAME
- TAG_END = /\/?>/
- HEX = /[0-9a-fA-F]/
- ENTITY = /
- &
- (?:
- \w+
- |
- \#
- (?:
- \d+
- |
- x#{HEX}+
- )
- )
- ;
- /ox
-
- PLAIN_STRING_CONTENT = {
- "'" => /[^&'>\n]+/,
- '"' => /[^&">\n]+/,
- }
-
- def reset
- super
- @state = :initial
- end
-
- private
- def setup
- @state = :initial
- @plain_string_content = nil
- end
-
- def scan_tokens tokens, options
-
- state = @state
- plain_string_content = @plain_string_content
-
- until eos?
-
- kind = nil
- match = nil
-
- if scan(/\s+/m)
- kind = :space
-
- else
-
- case state
-
- when :initial
- if scan(/<!--.*?-->/m)
- kind = :comment
- elsif scan(/<!DOCTYPE.*?>/m)
- kind = :doctype
- elsif scan(/<\?xml.*?\?>/m)
- kind = :preprocessor
- elsif scan(/<\?.*?\?>|<%.*?%>/m)
- kind = :comment
- elsif scan(/<\/[-\w.:]*>/m)
- kind = :tag
- elsif match = scan(/<[-\w.:]+>?/m)
- kind = :tag
- state = :attribute unless match[-1] == ?>
- elsif scan(/[^<>&]+/)
- kind = :plain
- elsif scan(/#{ENTITY}/ox)
- kind = :entity
- elsif scan(/[<>&]/)
- kind = :error
- else
- raise_inspect '[BUG] else-case reached with state %p' % [state], tokens
- end
-
- when :attribute
- if scan(/#{TAG_END}/)
- kind = :tag
- state = :initial
- elsif scan(/#{ATTR_NAME}/o)
- kind = :attribute_name
- state = :attribute_equal
- else
- kind = :error
- getch
- end
-
- when :attribute_equal
- if scan(/=/)
- kind = :operator
- state = :attribute_value
- elsif scan(/#{ATTR_NAME}/o)
- kind = :attribute_name
- elsif scan(/#{TAG_END}/o)
- kind = :tag
- state = :initial
- elsif scan(/./)
- kind = :error
- state = :attribute
- end
-
- when :attribute_value
- if scan(/#{ATTR_VALUE_UNQUOTED}/o)
- kind = :attribute_value
- state = :attribute
- elsif match = scan(/["']/)
- tokens << [:open, :string]
- state = :attribute_value_string
- plain_string_content = PLAIN_STRING_CONTENT[match]
- kind = :delimiter
- elsif scan(/#{TAG_END}/o)
- kind = :tag
- state = :initial
- else
- kind = :error
- getch
- end
-
- when :attribute_value_string
- if scan(plain_string_content)
- kind = :content
- elsif scan(/['"]/)
- tokens << [matched, :delimiter]
- tokens << [:close, :string]
- state = :attribute
- next
- elsif scan(/#{ENTITY}/ox)
- kind = :entity
- elsif scan(/&/)
- kind = :content
- elsif scan(/[\n>]/)
- tokens << [:close, :string]
- kind = :error
- state = :initial
- end
-
- else
- raise_inspect 'Unknown state: %p' % [state], tokens
-
- end
-
- end
-
- match ||= matched
- if $CODERAY_DEBUG and not kind
- raise_inspect 'Error token %p in line %d' %
- [[match, kind], line], tokens, state
- end
- raise_inspect 'Empty token', tokens unless match
-
- tokens << [match, kind]
- end
-
- if options[:keep_state]
- @state = state
- @plain_string_content = plain_string_content
- end
-
- tokens
- end
-
- end
-
-end
-end
+++ /dev/null
-module CodeRay
-module Scanners
-
- class Java < Scanner
-
- include Streamable
- register_for :java
- helper :builtin_types
-
- # http://java.sun.com/docs/books/tutorial/java/nutsandbolts/_keywords.html
- KEYWORDS = %w[
- assert break case catch continue default do else
- finally for if instanceof import new package
- return switch throw try typeof while
- debugger export
- ]
- RESERVED = %w[ const goto ]
- CONSTANTS = %w[ false null true ]
- MAGIC_VARIABLES = %w[ this super ]
- TYPES = %w[
- boolean byte char class double enum float int interface long
- short void
- ] << '[]' # because int[] should be highlighted as a type
- DIRECTIVES = %w[
- abstract extends final implements native private protected public
- static strictfp synchronized throws transient volatile
- ]
-
- IDENT_KIND = WordList.new(:ident).
- add(KEYWORDS, :keyword).
- add(RESERVED, :reserved).
- add(CONSTANTS, :pre_constant).
- add(MAGIC_VARIABLES, :local_variable).
- add(TYPES, :type).
- add(BuiltinTypes::List, :pre_type).
- add(BuiltinTypes::List.select { |builtin| builtin[/(Error|Exception)$/] }, :exception).
- add(DIRECTIVES, :directive)
-
- ESCAPE = / [bfnrtv\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} /x
- UNICODE_ESCAPE = / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} /x
- STRING_CONTENT_PATTERN = {
- "'" => /[^\\']+/,
- '"' => /[^\\"]+/,
- '/' => /[^\\\/]+/,
- }
- IDENT = /[a-zA-Z_][A-Za-z_0-9]*/
-
- def scan_tokens tokens, options
-
- state = :initial
- string_delimiter = nil
- import_clause = class_name_follows = last_token_dot = false
-
- until eos?
-
- kind = nil
- match = nil
-
- case state
-
- when :initial
-
- if match = scan(/ \s+ | \\\n /x)
- tokens << [match, :space]
- next
-
- elsif match = scan(%r! // [^\n\\]* (?: \\. [^\n\\]* )* | /\* (?: .*? \*/ | .* ) !mx)
- tokens << [match, :comment]
- next
-
- elsif import_clause && scan(/ #{IDENT} (?: \. #{IDENT} )* /ox)
- kind = :include
-
- elsif match = scan(/ #{IDENT} | \[\] /ox)
- kind = IDENT_KIND[match]
- if last_token_dot
- kind = :ident
- elsif class_name_follows
- kind = :class
- class_name_follows = false
- else
- import_clause = true if match == 'import'
- class_name_follows = true if match == 'class' || match == 'interface'
- end
-
- elsif scan(/ \.(?!\d) | [,?:()\[\]}] | -- | \+\+ | && | \|\| | \*\*=? | [-+*\/%^~&|<>=!]=? | <<<?=? | >>>?=? /x)
- kind = :operator
-
- elsif scan(/;/)
- import_clause = false
- kind = :operator
-
- elsif scan(/\{/)
- class_name_follows = false
- kind = :operator
-
- elsif check(/[\d.]/)
- if scan(/0[xX][0-9A-Fa-f]+/)
- kind = :hex
- elsif scan(/(?>0[0-7]+)(?![89.eEfF])/)
- kind = :oct
- elsif scan(/\d+[fFdD]|\d*\.\d+(?:[eE][+-]?\d+)?[fFdD]?|\d+[eE][+-]?\d+[fFdD]?/)
- kind = :float
- elsif scan(/\d+[lL]?/)
- kind = :integer
- end
-
- elsif match = scan(/["']/)
- tokens << [:open, :string]
- state = :string
- string_delimiter = match
- kind = :delimiter
-
- elsif scan(/ @ #{IDENT} /ox)
- kind = :annotation
-
- else
- getch
- kind = :error
-
- end
-
- when :string
- if scan(STRING_CONTENT_PATTERN[string_delimiter])
- kind = :content
- elsif match = scan(/["'\/]/)
- tokens << [match, :delimiter]
- tokens << [:close, state]
- string_delimiter = nil
- state = :initial
- next
- elsif state == :string && (match = scan(/ \\ (?: #{ESCAPE} | #{UNICODE_ESCAPE} ) /mox))
- if string_delimiter == "'" && !(match == "\\\\" || match == "\\'")
- kind = :content
- else
- kind = :char
- end
- elsif scan(/\\./m)
- kind = :content
- elsif scan(/ \\ | $ /x)
- tokens << [:close, :delimiter]
- kind = :error
- state = :initial
- else
- raise_inspect "else case \" reached; %p not handled." % peek(1), tokens
- end
-
- else
- raise_inspect 'Unknown state', tokens
-
- end
-
- match ||= matched
- if $CODERAY_DEBUG and not kind
- raise_inspect 'Error token %p in line %d' %
- [[match, kind], line], tokens
- end
- raise_inspect 'Empty token', tokens unless match
-
- last_token_dot = match == '.'
-
- tokens << [match, kind]
-
- end
-
- if state == :string
- tokens << [:close, state]
- end
-
- tokens
- end
-
- end
-
-end
-end
+++ /dev/null
-module CodeRay
-module Scanners
-
- module Java::BuiltinTypes # :nodoc:
-
- List = %w[
- AbstractAction AbstractBorder AbstractButton AbstractCellEditor AbstractCollection
- AbstractColorChooserPanel AbstractDocument AbstractExecutorService AbstractInterruptibleChannel
- AbstractLayoutCache AbstractList AbstractListModel AbstractMap AbstractMethodError AbstractPreferences
- AbstractQueue AbstractQueuedSynchronizer AbstractSelectableChannel AbstractSelectionKey AbstractSelector
- AbstractSequentialList AbstractSet AbstractSpinnerModel AbstractTableModel AbstractUndoableEdit
- AbstractWriter AccessControlContext AccessControlException AccessController AccessException Accessible
- AccessibleAction AccessibleAttributeSequence AccessibleBundle AccessibleComponent AccessibleContext
- AccessibleEditableText AccessibleExtendedComponent AccessibleExtendedTable AccessibleExtendedText
- AccessibleHyperlink AccessibleHypertext AccessibleIcon AccessibleKeyBinding AccessibleObject
- AccessibleRelation AccessibleRelationSet AccessibleResourceBundle AccessibleRole AccessibleSelection
- AccessibleState AccessibleStateSet AccessibleStreamable AccessibleTable AccessibleTableModelChange
- AccessibleText AccessibleTextSequence AccessibleValue AccountException AccountExpiredException
- AccountLockedException AccountNotFoundException Acl AclEntry AclNotFoundException Action ActionEvent
- ActionListener ActionMap ActionMapUIResource Activatable ActivateFailedException ActivationDesc
- ActivationException ActivationGroup ActivationGroupDesc ActivationGroupID ActivationGroup_Stub
- ActivationID ActivationInstantiator ActivationMonitor ActivationSystem Activator ActiveEvent
- ActivityCompletedException ActivityRequiredException Adjustable AdjustmentEvent AdjustmentListener
- Adler32 AffineTransform AffineTransformOp AlgorithmParameterGenerator AlgorithmParameterGeneratorSpi
- AlgorithmParameters AlgorithmParameterSpec AlgorithmParametersSpi AllPermission AlphaComposite
- AlreadyBoundException AlreadyConnectedException AncestorEvent AncestorListener AnnotatedElement
- Annotation AnnotationFormatError AnnotationTypeMismatchException AppConfigurationEntry Appendable Applet
- AppletContext AppletInitializer AppletStub Arc2D Area AreaAveragingScaleFilter ArithmeticException Array
- ArrayBlockingQueue ArrayIndexOutOfBoundsException ArrayList Arrays ArrayStoreException ArrayType
- AssertionError AsyncBoxView AsynchronousCloseException AtomicBoolean AtomicInteger AtomicIntegerArray
- AtomicIntegerFieldUpdater AtomicLong AtomicLongArray AtomicLongFieldUpdater AtomicMarkableReference
- AtomicReference AtomicReferenceArray AtomicReferenceFieldUpdater AtomicStampedReference Attribute
- AttributeChangeNotification AttributeChangeNotificationFilter AttributedCharacterIterator
- AttributedString AttributeException AttributeInUseException AttributeList AttributeModificationException
- AttributeNotFoundException Attributes AttributeSet AttributeSetUtilities AttributeValueExp AudioClip
- AudioFileFormat AudioFileReader AudioFileWriter AudioFormat AudioInputStream AudioPermission AudioSystem
- AuthenticationException AuthenticationNotSupportedException Authenticator AuthorizeCallback
- AuthPermission AuthProvider Autoscroll AWTError AWTEvent AWTEventListener AWTEventListenerProxy
- AWTEventMulticaster AWTException AWTKeyStroke AWTPermission BackingStoreException
- BadAttributeValueExpException BadBinaryOpValueExpException BadLocationException BadPaddingException
- BadStringOperationException BandCombineOp BandedSampleModel BaseRowSet BasicArrowButton BasicAttribute
- BasicAttributes BasicBorders BasicButtonListener BasicButtonUI BasicCheckBoxMenuItemUI BasicCheckBoxUI
- BasicColorChooserUI BasicComboBoxEditor BasicComboBoxRenderer BasicComboBoxUI BasicComboPopup
- BasicControl BasicDesktopIconUI BasicDesktopPaneUI BasicDirectoryModel BasicEditorPaneUI
- BasicFileChooserUI BasicFormattedTextFieldUI BasicGraphicsUtils BasicHTML BasicIconFactory
- BasicInternalFrameTitlePane BasicInternalFrameUI BasicLabelUI BasicListUI BasicLookAndFeel
- BasicMenuBarUI BasicMenuItemUI BasicMenuUI BasicOptionPaneUI BasicPanelUI BasicPasswordFieldUI
- BasicPermission BasicPopupMenuSeparatorUI BasicPopupMenuUI BasicProgressBarUI BasicRadioButtonMenuItemUI
- BasicRadioButtonUI BasicRootPaneUI BasicScrollBarUI BasicScrollPaneUI BasicSeparatorUI BasicSliderUI
- BasicSpinnerUI BasicSplitPaneDivider BasicSplitPaneUI BasicStroke BasicTabbedPaneUI BasicTableHeaderUI
- BasicTableUI BasicTextAreaUI BasicTextFieldUI BasicTextPaneUI BasicTextUI BasicToggleButtonUI
- BasicToolBarSeparatorUI BasicToolBarUI BasicToolTipUI BasicTreeUI BasicViewportUI BatchUpdateException
- BeanContext BeanContextChild BeanContextChildComponentProxy BeanContextChildSupport
- BeanContextContainerProxy BeanContextEvent BeanContextMembershipEvent BeanContextMembershipListener
- BeanContextProxy BeanContextServiceAvailableEvent BeanContextServiceProvider
- BeanContextServiceProviderBeanInfo BeanContextServiceRevokedEvent BeanContextServiceRevokedListener
- BeanContextServices BeanContextServicesListener BeanContextServicesSupport BeanContextSupport
- BeanDescriptor BeanInfo Beans BevelBorder Bidi BigDecimal BigInteger BinaryRefAddr BindException Binding
- BitSet Blob BlockingQueue BlockView BMPImageWriteParam Book Boolean BooleanControl Border BorderFactory
- BorderLayout BorderUIResource BoundedRangeModel Box BoxLayout BoxView BreakIterator
- BrokenBarrierException Buffer BufferCapabilities BufferedImage BufferedImageFilter BufferedImageOp
- BufferedInputStream BufferedOutputStream BufferedReader BufferedWriter BufferOverflowException
- BufferStrategy BufferUnderflowException Button ButtonGroup ButtonModel ButtonUI Byte
- ByteArrayInputStream ByteArrayOutputStream ByteBuffer ByteChannel ByteLookupTable ByteOrder CachedRowSet
- CacheRequest CacheResponse Calendar Callable CallableStatement Callback CallbackHandler
- CancelablePrintJob CancellationException CancelledKeyException CannotProceedException
- CannotRedoException CannotUndoException Canvas CardLayout Caret CaretEvent CaretListener CellEditor
- CellEditorListener CellRendererPane Certificate CertificateEncodingException CertificateException
- CertificateExpiredException CertificateFactory CertificateFactorySpi CertificateNotYetValidException
- CertificateParsingException CertPath CertPathBuilder CertPathBuilderException CertPathBuilderResult
- CertPathBuilderSpi CertPathParameters CertPathTrustManagerParameters CertPathValidator
- CertPathValidatorException CertPathValidatorResult CertPathValidatorSpi CertSelector CertStore
- CertStoreException CertStoreParameters CertStoreSpi ChangedCharSetException ChangeEvent ChangeListener
- Channel Channels Character CharacterCodingException CharacterIterator CharArrayReader CharArrayWriter
- CharBuffer CharConversionException CharSequence Charset CharsetDecoder CharsetEncoder CharsetProvider
- Checkbox CheckboxGroup CheckboxMenuItem CheckedInputStream CheckedOutputStream Checksum Choice
- ChoiceCallback ChoiceFormat Chromaticity Cipher CipherInputStream CipherOutputStream CipherSpi Class
- ClassCastException ClassCircularityError ClassDefinition ClassDesc ClassFileTransformer ClassFormatError
- ClassLoader ClassLoaderRepository ClassLoadingMXBean ClassNotFoundException Clip Clipboard
- ClipboardOwner Clob Cloneable CloneNotSupportedException Closeable ClosedByInterruptException
- ClosedChannelException ClosedSelectorException CMMException CoderMalfunctionError CoderResult CodeSigner
- CodeSource CodingErrorAction CollationElementIterator CollationKey Collator Collection
- CollectionCertStoreParameters Collections Color ColorChooserComponentFactory ColorChooserUI
- ColorConvertOp ColorModel ColorSelectionModel ColorSpace ColorSupported ColorType ColorUIResource
- ComboBoxEditor ComboBoxModel ComboBoxUI ComboPopup CommunicationException Comparable Comparator
- CompilationMXBean Compiler CompletionService Component ComponentAdapter ComponentColorModel
- ComponentEvent ComponentInputMap ComponentInputMapUIResource ComponentListener ComponentOrientation
- ComponentSampleModel ComponentUI ComponentView Composite CompositeContext CompositeData
- CompositeDataSupport CompositeName CompositeType CompositeView CompoundBorder CompoundControl
- CompoundEdit CompoundName Compression ConcurrentHashMap ConcurrentLinkedQueue ConcurrentMap
- ConcurrentModificationException Condition Configuration ConfigurationException ConfirmationCallback
- ConnectException ConnectIOException Connection ConnectionEvent ConnectionEventListener
- ConnectionPendingException ConnectionPoolDataSource ConsoleHandler Constructor Container
- ContainerAdapter ContainerEvent ContainerListener ContainerOrderFocusTraversalPolicy ContentHandler
- ContentHandlerFactory ContentModel Context ContextNotEmptyException ContextualRenderedImageFactory
- Control ControlFactory ControllerEventListener ConvolveOp CookieHandler Copies CopiesSupported
- CopyOnWriteArrayList CopyOnWriteArraySet CountDownLatch CounterMonitor CounterMonitorMBean CRC32
- CredentialException CredentialExpiredException CredentialNotFoundException CRL CRLException CRLSelector
- CropImageFilter CSS CubicCurve2D Currency Cursor Customizer CyclicBarrier DatabaseMetaData DataBuffer
- DataBufferByte DataBufferDouble DataBufferFloat DataBufferInt DataBufferShort DataBufferUShort
- DataFlavor DataFormatException DatagramChannel DatagramPacket DatagramSocket DatagramSocketImpl
- DatagramSocketImplFactory DataInput DataInputStream DataLine DataOutput DataOutputStream DataSource
- DataTruncation DatatypeConfigurationException DatatypeConstants DatatypeFactory Date DateFormat
- DateFormatSymbols DateFormatter DateTimeAtCompleted DateTimeAtCreation DateTimeAtProcessing
- DateTimeSyntax DebugGraphics DecimalFormat DecimalFormatSymbols DefaultBoundedRangeModel
- DefaultButtonModel DefaultCaret DefaultCellEditor DefaultColorSelectionModel DefaultComboBoxModel
- DefaultDesktopManager DefaultEditorKit DefaultFocusManager DefaultFocusTraversalPolicy DefaultFormatter
- DefaultFormatterFactory DefaultHighlighter DefaultKeyboardFocusManager DefaultListCellRenderer
- DefaultListModel DefaultListSelectionModel DefaultLoaderRepository DefaultMenuLayout DefaultMetalTheme
- DefaultMutableTreeNode DefaultPersistenceDelegate DefaultSingleSelectionModel DefaultStyledDocument
- DefaultTableCellRenderer DefaultTableColumnModel DefaultTableModel DefaultTextUI DefaultTreeCellEditor
- DefaultTreeCellRenderer DefaultTreeModel DefaultTreeSelectionModel Deflater DeflaterOutputStream Delayed
- DelayQueue DelegationPermission Deprecated Descriptor DescriptorAccess DescriptorSupport DESedeKeySpec
- DesignMode DESKeySpec DesktopIconUI DesktopManager DesktopPaneUI Destination Destroyable
- DestroyFailedException DGC DHGenParameterSpec DHKey DHParameterSpec DHPrivateKey DHPrivateKeySpec
- DHPublicKey DHPublicKeySpec Dialog Dictionary DigestException DigestInputStream DigestOutputStream
- Dimension Dimension2D DimensionUIResource DirContext DirectColorModel DirectoryManager DirObjectFactory
- DirStateFactory DisplayMode DnDConstants Doc DocAttribute DocAttributeSet DocFlavor DocPrintJob Document
- DocumentBuilder DocumentBuilderFactory Documented DocumentEvent DocumentFilter DocumentListener
- DocumentName DocumentParser DomainCombiner DOMLocator DOMResult DOMSource Double DoubleBuffer
- DragGestureEvent DragGestureListener DragGestureRecognizer DragSource DragSourceAdapter
- DragSourceContext DragSourceDragEvent DragSourceDropEvent DragSourceEvent DragSourceListener
- DragSourceMotionListener Driver DriverManager DriverPropertyInfo DropTarget DropTargetAdapter
- DropTargetContext DropTargetDragEvent DropTargetDropEvent DropTargetEvent DropTargetListener DSAKey
- DSAKeyPairGenerator DSAParameterSpec DSAParams DSAPrivateKey DSAPrivateKeySpec DSAPublicKey
- DSAPublicKeySpec DTD DTDConstants DuplicateFormatFlagsException Duration DynamicMBean ECField ECFieldF2m
- ECFieldFp ECGenParameterSpec ECKey ECParameterSpec ECPoint ECPrivateKey ECPrivateKeySpec ECPublicKey
- ECPublicKeySpec EditorKit Element ElementIterator ElementType Ellipse2D EllipticCurve EmptyBorder
- EmptyStackException EncodedKeySpec Encoder EncryptedPrivateKeyInfo Entity Enum
- EnumConstantNotPresentException EnumControl Enumeration EnumMap EnumSet EnumSyntax EOFException Error
- ErrorListener ErrorManager EtchedBorder Event EventContext EventDirContext EventHandler EventListener
- EventListenerList EventListenerProxy EventObject EventQueue EventSetDescriptor Exception
- ExceptionInInitializerError ExceptionListener Exchanger ExecutionException Executor
- ExecutorCompletionService Executors ExecutorService ExemptionMechanism ExemptionMechanismException
- ExemptionMechanismSpi ExpandVetoException ExportException Expression ExtendedRequest ExtendedResponse
- Externalizable FactoryConfigurationError FailedLoginException FeatureDescriptor Fidelity Field
- FieldPosition FieldView File FileCacheImageInputStream FileCacheImageOutputStream FileChannel
- FileChooserUI FileDescriptor FileDialog FileFilter FileHandler FileImageInputStream
- FileImageOutputStream FileInputStream FileLock FileLockInterruptionException FilenameFilter FileNameMap
- FileNotFoundException FileOutputStream FilePermission FileReader FileSystemView FileView FileWriter
- Filter FilteredImageSource FilteredRowSet FilterInputStream FilterOutputStream FilterReader FilterWriter
- Finishings FixedHeightLayoutCache FlatteningPathIterator FlavorEvent FlavorException FlavorListener
- FlavorMap FlavorTable Float FloatBuffer FloatControl FlowLayout FlowView Flushable FocusAdapter
- FocusEvent FocusListener FocusManager FocusTraversalPolicy Font FontFormatException FontMetrics
- FontRenderContext FontUIResource Format FormatConversionProvider FormatFlagsConversionMismatchException
- Formattable FormattableFlags Formatter FormatterClosedException FormSubmitEvent FormView Frame Future
- FutureTask GapContent GarbageCollectorMXBean GatheringByteChannel GaugeMonitor GaugeMonitorMBean
- GeneralPath GeneralSecurityException GenericArrayType GenericDeclaration GenericSignatureFormatError
- GlyphJustificationInfo GlyphMetrics GlyphVector GlyphView GradientPaint GraphicAttribute Graphics
- Graphics2D GraphicsConfigTemplate GraphicsConfiguration GraphicsDevice GraphicsEnvironment GrayFilter
- GregorianCalendar GridBagConstraints GridBagLayout GridLayout Group Guard GuardedObject GZIPInputStream
- GZIPOutputStream Handler HandshakeCompletedEvent HandshakeCompletedListener HasControls HashAttributeSet
- HashDocAttributeSet HashMap HashPrintJobAttributeSet HashPrintRequestAttributeSet
- HashPrintServiceAttributeSet HashSet Hashtable HeadlessException HierarchyBoundsAdapter
- HierarchyBoundsListener HierarchyEvent HierarchyListener Highlighter HostnameVerifier HTML HTMLDocument
- HTMLEditorKit HTMLFrameHyperlinkEvent HTMLWriter HttpRetryException HttpsURLConnection HttpURLConnection
- HyperlinkEvent HyperlinkListener ICC_ColorSpace ICC_Profile ICC_ProfileGray ICC_ProfileRGB Icon
- IconUIResource IconView Identity IdentityHashMap IdentityScope IIOByteBuffer IIOException IIOImage
- IIOInvalidTreeException IIOMetadata IIOMetadataController IIOMetadataFormat IIOMetadataFormatImpl
- IIOMetadataNode IIOParam IIOParamController IIOReadProgressListener IIOReadUpdateListener
- IIOReadWarningListener IIORegistry IIOServiceProvider IIOWriteProgressListener IIOWriteWarningListener
- IllegalAccessError IllegalAccessException IllegalArgumentException IllegalBlockingModeException
- IllegalBlockSizeException IllegalCharsetNameException IllegalClassFormatException
- IllegalComponentStateException IllegalFormatCodePointException IllegalFormatConversionException
- IllegalFormatException IllegalFormatFlagsException IllegalFormatPrecisionException
- IllegalFormatWidthException IllegalMonitorStateException IllegalPathStateException
- IllegalSelectorException IllegalStateException IllegalThreadStateException Image ImageCapabilities
- ImageConsumer ImageFilter ImageGraphicAttribute ImageIcon ImageInputStream ImageInputStreamImpl
- ImageInputStreamSpi ImageIO ImageObserver ImageOutputStream ImageOutputStreamImpl ImageOutputStreamSpi
- ImageProducer ImageReader ImageReaderSpi ImageReaderWriterSpi ImageReadParam ImageTranscoder
- ImageTranscoderSpi ImageTypeSpecifier ImageView ImageWriteParam ImageWriter ImageWriterSpi
- ImagingOpException IncompatibleClassChangeError IncompleteAnnotationException IndexColorModel
- IndexedPropertyChangeEvent IndexedPropertyDescriptor IndexOutOfBoundsException Inet4Address Inet6Address
- InetAddress InetSocketAddress Inflater InflaterInputStream InheritableThreadLocal Inherited
- InitialContext InitialContextFactory InitialContextFactoryBuilder InitialDirContext InitialLdapContext
- InlineView InputContext InputEvent InputMap InputMapUIResource InputMethod InputMethodContext
- InputMethodDescriptor InputMethodEvent InputMethodHighlight InputMethodListener InputMethodRequests
- InputMismatchException InputStream InputStreamReader InputSubset InputVerifier Insets InsetsUIResource
- InstanceAlreadyExistsException InstanceNotFoundException InstantiationError InstantiationException
- Instrument Instrumentation InsufficientResourcesException IntBuffer Integer IntegerSyntax InternalError
- InternalFrameAdapter InternalFrameEvent InternalFrameFocusTraversalPolicy InternalFrameListener
- InternalFrameUI InternationalFormatter InterruptedException InterruptedIOException
- InterruptedNamingException InterruptibleChannel IntrospectionException Introspector
- InvalidActivityException InvalidAlgorithmParameterException InvalidApplicationException
- InvalidAttributeIdentifierException InvalidAttributesException InvalidAttributeValueException
- InvalidClassException InvalidDnDOperationException InvalidKeyException InvalidKeySpecException
- InvalidMarkException InvalidMidiDataException InvalidNameException InvalidObjectException
- InvalidOpenTypeException InvalidParameterException InvalidParameterSpecException
- InvalidPreferencesFormatException InvalidPropertiesFormatException InvalidRelationIdException
- InvalidRelationServiceException InvalidRelationTypeException InvalidRoleInfoException
- InvalidRoleValueException InvalidSearchControlsException InvalidSearchFilterException
- InvalidTargetObjectTypeException InvalidTransactionException InvocationEvent InvocationHandler
- InvocationTargetException IOException ItemEvent ItemListener ItemSelectable Iterable Iterator
- IvParameterSpec JApplet JarEntry JarException JarFile JarInputStream JarOutputStream JarURLConnection
- JButton JCheckBox JCheckBoxMenuItem JColorChooser JComboBox JComponent JdbcRowSet JDesktopPane JDialog
- JEditorPane JFileChooser JFormattedTextField JFrame JInternalFrame JLabel JLayeredPane JList JMenu
- JMenuBar JMenuItem JMException JMRuntimeException JMXAuthenticator JMXConnectionNotification
- JMXConnector JMXConnectorFactory JMXConnectorProvider JMXConnectorServer JMXConnectorServerFactory
- JMXConnectorServerMBean JMXConnectorServerProvider JMXPrincipal JMXProviderException
- JMXServerErrorException JMXServiceURL JobAttributes JobHoldUntil JobImpressions JobImpressionsCompleted
- JobImpressionsSupported JobKOctets JobKOctetsProcessed JobKOctetsSupported JobMediaSheets
- JobMediaSheetsCompleted JobMediaSheetsSupported JobMessageFromOperator JobName JobOriginatingUserName
- JobPriority JobPrioritySupported JobSheets JobState JobStateReason JobStateReasons Joinable JoinRowSet
- JOptionPane JPanel JPasswordField JPEGHuffmanTable JPEGImageReadParam JPEGImageWriteParam JPEGQTable
- JPopupMenu JProgressBar JRadioButton JRadioButtonMenuItem JRootPane JScrollBar JScrollPane JSeparator
- JSlider JSpinner JSplitPane JTabbedPane JTable JTableHeader JTextArea JTextComponent JTextField
- JTextPane JToggleButton JToolBar JToolTip JTree JViewport JWindow KerberosKey KerberosPrincipal
- KerberosTicket Kernel Key KeyAdapter KeyAgreement KeyAgreementSpi KeyAlreadyExistsException
- KeyboardFocusManager KeyEvent KeyEventDispatcher KeyEventPostProcessor KeyException KeyFactory
- KeyFactorySpi KeyGenerator KeyGeneratorSpi KeyListener KeyManagementException KeyManager
- KeyManagerFactory KeyManagerFactorySpi Keymap KeyPair KeyPairGenerator KeyPairGeneratorSpi KeyRep
- KeySpec KeyStore KeyStoreBuilderParameters KeyStoreException KeyStoreSpi KeyStroke Label LabelUI
- LabelView LanguageCallback LastOwnerException LayeredHighlighter LayoutFocusTraversalPolicy
- LayoutManager LayoutManager2 LayoutQueue LDAPCertStoreParameters LdapContext LdapName
- LdapReferralException Lease Level LimitExceededException Line Line2D LineBorder LineBreakMeasurer
- LineEvent LineListener LineMetrics LineNumberInputStream LineNumberReader LineUnavailableException
- LinkageError LinkedBlockingQueue LinkedHashMap LinkedHashSet LinkedList LinkException LinkLoopException
- LinkRef List ListCellRenderer ListDataEvent ListDataListener ListenerNotFoundException ListIterator
- ListModel ListResourceBundle ListSelectionEvent ListSelectionListener ListSelectionModel ListUI ListView
- LoaderHandler Locale LocateRegistry Lock LockSupport Logger LoggingMXBean LoggingPermission LoginContext
- LoginException LoginModule LogManager LogRecord LogStream Long LongBuffer LookAndFeel LookupOp
- LookupTable Mac MacSpi MalformedInputException MalformedLinkException MalformedObjectNameException
- MalformedParameterizedTypeException MalformedURLException ManagementFactory ManagementPermission
- ManageReferralControl ManagerFactoryParameters Manifest Map MappedByteBuffer MarshalException
- MarshalledObject MaskFormatter Matcher MatchResult Math MathContext MatteBorder MBeanAttributeInfo
- MBeanConstructorInfo MBeanException MBeanFeatureInfo MBeanInfo MBeanNotificationInfo MBeanOperationInfo
- MBeanParameterInfo MBeanPermission MBeanRegistration MBeanRegistrationException MBeanServer
- MBeanServerBuilder MBeanServerConnection MBeanServerDelegate MBeanServerDelegateMBean MBeanServerFactory
- MBeanServerForwarder MBeanServerInvocationHandler MBeanServerNotification MBeanServerNotificationFilter
- MBeanServerPermission MBeanTrustPermission Media MediaName MediaPrintableArea MediaSize MediaSizeName
- MediaTracker MediaTray Member MemoryCacheImageInputStream MemoryCacheImageOutputStream MemoryHandler
- MemoryImageSource MemoryManagerMXBean MemoryMXBean MemoryNotificationInfo MemoryPoolMXBean MemoryType
- MemoryUsage Menu MenuBar MenuBarUI MenuComponent MenuContainer MenuDragMouseEvent MenuDragMouseListener
- MenuElement MenuEvent MenuItem MenuItemUI MenuKeyEvent MenuKeyListener MenuListener MenuSelectionManager
- MenuShortcut MessageDigest MessageDigestSpi MessageFormat MetaEventListener MetalBorders MetalButtonUI
- MetalCheckBoxIcon MetalCheckBoxUI MetalComboBoxButton MetalComboBoxEditor MetalComboBoxIcon
- MetalComboBoxUI MetalDesktopIconUI MetalFileChooserUI MetalIconFactory MetalInternalFrameTitlePane
- MetalInternalFrameUI MetalLabelUI MetalLookAndFeel MetalMenuBarUI MetalPopupMenuSeparatorUI
- MetalProgressBarUI MetalRadioButtonUI MetalRootPaneUI MetalScrollBarUI MetalScrollButton
- MetalScrollPaneUI MetalSeparatorUI MetalSliderUI MetalSplitPaneUI MetalTabbedPaneUI MetalTextFieldUI
- MetalTheme MetalToggleButtonUI MetalToolBarUI MetalToolTipUI MetalTreeUI MetaMessage Method
- MethodDescriptor MGF1ParameterSpec MidiChannel MidiDevice MidiDeviceProvider MidiEvent MidiFileFormat
- MidiFileReader MidiFileWriter MidiMessage MidiSystem MidiUnavailableException MimeTypeParseException
- MinimalHTMLWriter MissingFormatArgumentException MissingFormatWidthException MissingResourceException
- Mixer MixerProvider MLet MLetMBean ModelMBean ModelMBeanAttributeInfo ModelMBeanConstructorInfo
- ModelMBeanInfo ModelMBeanInfoSupport ModelMBeanNotificationBroadcaster ModelMBeanNotificationInfo
- ModelMBeanOperationInfo ModificationItem Modifier Monitor MonitorMBean MonitorNotification
- MonitorSettingException MouseAdapter MouseDragGestureRecognizer MouseEvent MouseInfo MouseInputAdapter
- MouseInputListener MouseListener MouseMotionAdapter MouseMotionListener MouseWheelEvent
- MouseWheelListener MultiButtonUI MulticastSocket MultiColorChooserUI MultiComboBoxUI MultiDesktopIconUI
- MultiDesktopPaneUI MultiDoc MultiDocPrintJob MultiDocPrintService MultiFileChooserUI
- MultiInternalFrameUI MultiLabelUI MultiListUI MultiLookAndFeel MultiMenuBarUI MultiMenuItemUI
- MultiOptionPaneUI MultiPanelUI MultiPixelPackedSampleModel MultipleDocumentHandling MultipleMaster
- MultiPopupMenuUI MultiProgressBarUI MultiRootPaneUI MultiScrollBarUI MultiScrollPaneUI MultiSeparatorUI
- MultiSliderUI MultiSpinnerUI MultiSplitPaneUI MultiTabbedPaneUI MultiTableHeaderUI MultiTableUI
- MultiTextUI MultiToolBarUI MultiToolTipUI MultiTreeUI MultiViewportUI MutableAttributeSet
- MutableComboBoxModel MutableTreeNode Name NameAlreadyBoundException NameCallback NameClassPair
- NameNotFoundException NameParser NamespaceChangeListener NamespaceContext Naming NamingEnumeration
- NamingEvent NamingException NamingExceptionEvent NamingListener NamingManager NamingSecurityException
- NavigationFilter NegativeArraySizeException NetPermission NetworkInterface NoClassDefFoundError
- NoConnectionPendingException NodeChangeEvent NodeChangeListener NoInitialContextException
- NoninvertibleTransformException NonReadableChannelException NonWritableChannelException
- NoPermissionException NoRouteToHostException NoSuchAlgorithmException NoSuchAttributeException
- NoSuchElementException NoSuchFieldError NoSuchFieldException NoSuchMethodError NoSuchMethodException
- NoSuchObjectException NoSuchPaddingException NoSuchProviderException NotActiveException
- NotBoundException NotCompliantMBeanException NotContextException Notification NotificationBroadcaster
- NotificationBroadcasterSupport NotificationEmitter NotificationFilter NotificationFilterSupport
- NotificationListener NotificationResult NotOwnerException NotSerializableException NotYetBoundException
- NotYetConnectedException NullCipher NullPointerException Number NumberFormat NumberFormatException
- NumberFormatter NumberOfDocuments NumberOfInterveningJobs NumberUp NumberUpSupported NumericShaper
- OAEPParameterSpec Object ObjectChangeListener ObjectFactory ObjectFactoryBuilder ObjectInput
- ObjectInputStream ObjectInputValidation ObjectInstance ObjectName ObjectOutput ObjectOutputStream
- ObjectStreamClass ObjectStreamConstants ObjectStreamException ObjectStreamField ObjectView ObjID
- Observable Observer OceanTheme OpenDataException OpenMBeanAttributeInfo OpenMBeanAttributeInfoSupport
- OpenMBeanConstructorInfo OpenMBeanConstructorInfoSupport OpenMBeanInfo OpenMBeanInfoSupport
- OpenMBeanOperationInfo OpenMBeanOperationInfoSupport OpenMBeanParameterInfo
- OpenMBeanParameterInfoSupport OpenType OperatingSystemMXBean Operation OperationNotSupportedException
- OperationsException Option OptionalDataException OptionPaneUI OrientationRequested OutOfMemoryError
- OutputDeviceAssigned OutputKeys OutputStream OutputStreamWriter OverlappingFileLockException
- OverlayLayout Override Owner Pack200 Package PackedColorModel Pageable PageAttributes
- PagedResultsControl PagedResultsResponseControl PageFormat PageRanges PagesPerMinute PagesPerMinuteColor
- Paint PaintContext PaintEvent Panel PanelUI Paper ParagraphView ParameterBlock ParameterDescriptor
- ParameterizedType ParameterMetaData ParseException ParsePosition Parser ParserConfigurationException
- ParserDelegator PartialResultException PasswordAuthentication PasswordCallback PasswordView Patch
- PathIterator Pattern PatternSyntaxException PBEKey PBEKeySpec PBEParameterSpec PDLOverrideSupported
- Permission PermissionCollection Permissions PersistenceDelegate PersistentMBean PhantomReference Pipe
- PipedInputStream PipedOutputStream PipedReader PipedWriter PixelGrabber PixelInterleavedSampleModel
- PKCS8EncodedKeySpec PKIXBuilderParameters PKIXCertPathBuilderResult PKIXCertPathChecker
- PKIXCertPathValidatorResult PKIXParameters PlainDocument PlainView Point Point2D PointerInfo Policy
- PolicyNode PolicyQualifierInfo Polygon PooledConnection Popup PopupFactory PopupMenu PopupMenuEvent
- PopupMenuListener PopupMenuUI Port PortableRemoteObject PortableRemoteObjectDelegate
- PortUnreachableException Position Predicate PreferenceChangeEvent PreferenceChangeListener Preferences
- PreferencesFactory PreparedStatement PresentationDirection Principal Printable PrinterAbortException
- PrinterException PrinterGraphics PrinterInfo PrinterIOException PrinterIsAcceptingJobs PrinterJob
- PrinterLocation PrinterMakeAndModel PrinterMessageFromOperator PrinterMoreInfo
- PrinterMoreInfoManufacturer PrinterName PrinterResolution PrinterState PrinterStateReason
- PrinterStateReasons PrinterURI PrintEvent PrintException PrintGraphics PrintJob PrintJobAdapter
- PrintJobAttribute PrintJobAttributeEvent PrintJobAttributeListener PrintJobAttributeSet PrintJobEvent
- PrintJobListener PrintQuality PrintRequestAttribute PrintRequestAttributeSet PrintService
- PrintServiceAttribute PrintServiceAttributeEvent PrintServiceAttributeListener PrintServiceAttributeSet
- PrintServiceLookup PrintStream PrintWriter PriorityBlockingQueue PriorityQueue PrivateClassLoader
- PrivateCredentialPermission PrivateKey PrivateMLet PrivilegedAction PrivilegedActionException
- PrivilegedExceptionAction Process ProcessBuilder ProfileDataException ProgressBarUI ProgressMonitor
- ProgressMonitorInputStream Properties PropertyChangeEvent PropertyChangeListener
- PropertyChangeListenerProxy PropertyChangeSupport PropertyDescriptor PropertyEditor
- PropertyEditorManager PropertyEditorSupport PropertyPermission PropertyResourceBundle
- PropertyVetoException ProtectionDomain ProtocolException Provider ProviderException Proxy ProxySelector
- PSource PSSParameterSpec PublicKey PushbackInputStream PushbackReader QName QuadCurve2D Query QueryEval
- QueryExp Queue QueuedJobCount Random RandomAccess RandomAccessFile Raster RasterFormatException RasterOp
- RC2ParameterSpec RC5ParameterSpec Rdn Readable ReadableByteChannel Reader ReadOnlyBufferException
- ReadWriteLock RealmCallback RealmChoiceCallback Receiver Rectangle Rectangle2D RectangularShape
- ReentrantLock ReentrantReadWriteLock Ref RefAddr Reference Referenceable ReferenceQueue
- ReferenceUriSchemesSupported ReferralException ReflectionException ReflectPermission Refreshable
- RefreshFailedException Region RegisterableService Registry RegistryHandler RejectedExecutionException
- RejectedExecutionHandler Relation RelationException RelationNotFoundException RelationNotification
- RelationService RelationServiceMBean RelationServiceNotRegisteredException RelationSupport
- RelationSupportMBean RelationType RelationTypeNotFoundException RelationTypeSupport Remote RemoteCall
- RemoteException RemoteObject RemoteObjectInvocationHandler RemoteRef RemoteServer RemoteStub
- RenderableImage RenderableImageOp RenderableImageProducer RenderContext RenderedImage
- RenderedImageFactory Renderer RenderingHints RepaintManager ReplicateScaleFilter RequestingUserName
- RequiredModelMBean RescaleOp ResolutionSyntax Resolver ResolveResult ResourceBundle ResponseCache Result
- ResultSet ResultSetMetaData Retention RetentionPolicy ReverbType RGBImageFilter RMIClassLoader
- RMIClassLoaderSpi RMIClientSocketFactory RMIConnection RMIConnectionImpl RMIConnectionImpl_Stub
- RMIConnector RMIConnectorServer RMIFailureHandler RMIIIOPServerImpl RMIJRMPServerImpl
- RMISecurityException RMISecurityManager RMIServer RMIServerImpl RMIServerImpl_Stub
- RMIServerSocketFactory RMISocketFactory Robot Role RoleInfo RoleInfoNotFoundException RoleList
- RoleNotFoundException RoleResult RoleStatus RoleUnresolved RoleUnresolvedList RootPaneContainer
- RootPaneUI RoundingMode RoundRectangle2D RowMapper RowSet RowSetEvent RowSetInternal RowSetListener
- RowSetMetaData RowSetMetaDataImpl RowSetReader RowSetWarning RowSetWriter RSAKey RSAKeyGenParameterSpec
- RSAMultiPrimePrivateCrtKey RSAMultiPrimePrivateCrtKeySpec RSAOtherPrimeInfo RSAPrivateCrtKey
- RSAPrivateCrtKeySpec RSAPrivateKey RSAPrivateKeySpec RSAPublicKey RSAPublicKeySpec RTFEditorKit
- RuleBasedCollator Runnable Runtime RuntimeErrorException RuntimeException RuntimeMBeanException
- RuntimeMXBean RuntimeOperationsException RuntimePermission SampleModel Sasl SaslClient SaslClientFactory
- SaslException SaslServer SaslServerFactory Savepoint SAXParser SAXParserFactory SAXResult SAXSource
- SAXTransformerFactory Scanner ScatteringByteChannel ScheduledExecutorService ScheduledFuture
- ScheduledThreadPoolExecutor Schema SchemaFactory SchemaFactoryLoader SchemaViolationException Scrollable
- Scrollbar ScrollBarUI ScrollPane ScrollPaneAdjustable ScrollPaneConstants ScrollPaneLayout ScrollPaneUI
- SealedObject SearchControls SearchResult SecretKey SecretKeyFactory SecretKeyFactorySpi SecretKeySpec
- SecureCacheResponse SecureClassLoader SecureRandom SecureRandomSpi Security SecurityException
- SecurityManager SecurityPermission Segment SelectableChannel SelectionKey Selector SelectorProvider
- Semaphore SeparatorUI Sequence SequenceInputStream Sequencer SerialArray SerialBlob SerialClob
- SerialDatalink SerialException Serializable SerializablePermission SerialJavaObject SerialRef
- SerialStruct ServerCloneException ServerError ServerException ServerNotActiveException ServerRef
- ServerRuntimeException ServerSocket ServerSocketChannel ServerSocketFactory ServiceNotFoundException
- ServicePermission ServiceRegistry ServiceUI ServiceUIFactory ServiceUnavailableException Set
- SetOfIntegerSyntax Severity Shape ShapeGraphicAttribute SheetCollate Short ShortBuffer
- ShortBufferException ShortLookupTable ShortMessage Sides Signature SignatureException SignatureSpi
- SignedObject Signer SimpleAttributeSet SimpleBeanInfo SimpleDateFormat SimpleDoc SimpleFormatter
- SimpleTimeZone SimpleType SinglePixelPackedSampleModel SingleSelectionModel Size2DSyntax
- SizeLimitExceededException SizeRequirements SizeSequence Skeleton SkeletonMismatchException
- SkeletonNotFoundException SliderUI Socket SocketAddress SocketChannel SocketException SocketFactory
- SocketHandler SocketImpl SocketImplFactory SocketOptions SocketPermission SocketSecurityException
- SocketTimeoutException SoftBevelBorder SoftReference SortControl SortedMap SortedSet
- SortingFocusTraversalPolicy SortKey SortResponseControl Soundbank SoundbankReader SoundbankResource
- Source SourceDataLine SourceLocator SpinnerDateModel SpinnerListModel SpinnerModel SpinnerNumberModel
- SpinnerUI SplitPaneUI Spring SpringLayout SQLData SQLException SQLInput SQLInputImpl SQLOutput
- SQLOutputImpl SQLPermission SQLWarning SSLContext SSLContextSpi SSLEngine SSLEngineResult SSLException
- SSLHandshakeException SSLKeyException SSLPeerUnverifiedException SSLPermission SSLProtocolException
- SslRMIClientSocketFactory SslRMIServerSocketFactory SSLServerSocket SSLServerSocketFactory SSLSession
- SSLSessionBindingEvent SSLSessionBindingListener SSLSessionContext SSLSocket SSLSocketFactory Stack
- StackOverflowError StackTraceElement StandardMBean StartTlsRequest StartTlsResponse StateEdit
- StateEditable StateFactory Statement StreamCorruptedException StreamHandler StreamPrintService
- StreamPrintServiceFactory StreamResult StreamSource StreamTokenizer StrictMath String StringBuffer
- StringBufferInputStream StringBuilder StringCharacterIterator StringContent
- StringIndexOutOfBoundsException StringMonitor StringMonitorMBean StringReader StringRefAddr
- StringSelection StringTokenizer StringValueExp StringWriter Stroke Struct Stub StubDelegate
- StubNotFoundException Style StyleConstants StyleContext StyledDocument StyledEditorKit StyleSheet
- Subject SubjectDelegationPermission SubjectDomainCombiner SupportedValuesAttribute SuppressWarnings
- SwingConstants SwingPropertyChangeSupport SwingUtilities SyncFactory SyncFactoryException
- SyncFailedException SynchronousQueue SyncProvider SyncProviderException SyncResolver SynthConstants
- SynthContext Synthesizer SynthGraphicsUtils SynthLookAndFeel SynthPainter SynthStyle SynthStyleFactory
- SysexMessage System SystemColor SystemFlavorMap TabableView TabbedPaneUI TabExpander TableCellEditor
- TableCellRenderer TableColumn TableColumnModel TableColumnModelEvent TableColumnModelListener
- TableHeaderUI TableModel TableModelEvent TableModelListener TableUI TableView TabSet TabStop TabularData
- TabularDataSupport TabularType TagElement Target TargetDataLine TargetedNotification Templates
- TemplatesHandler TextAction TextArea TextAttribute TextComponent TextEvent TextField TextHitInfo
- TextInputCallback TextLayout TextListener TextMeasurer TextOutputCallback TextSyntax TextUI TexturePaint
- Thread ThreadDeath ThreadFactory ThreadGroup ThreadInfo ThreadLocal ThreadMXBean ThreadPoolExecutor
- Throwable Tie TileObserver Time TimeLimitExceededException TimeoutException Timer
- TimerAlarmClockNotification TimerMBean TimerNotification TimerTask Timestamp TimeUnit TimeZone
- TitledBorder ToolBarUI Toolkit ToolTipManager ToolTipUI TooManyListenersException Track
- TransactionalWriter TransactionRequiredException TransactionRolledbackException Transferable
- TransferHandler TransformAttribute Transformer TransformerConfigurationException TransformerException
- TransformerFactory TransformerFactoryConfigurationError TransformerHandler Transmitter Transparency
- TreeCellEditor TreeCellRenderer TreeExpansionEvent TreeExpansionListener TreeMap TreeModel
- TreeModelEvent TreeModelListener TreeNode TreePath TreeSelectionEvent TreeSelectionListener
- TreeSelectionModel TreeSet TreeUI TreeWillExpandListener TrustAnchor TrustManager TrustManagerFactory
- TrustManagerFactorySpi Type TypeInfoProvider TypeNotPresentException Types TypeVariable UID UIDefaults
- UIManager UIResource UndeclaredThrowableException UndoableEdit UndoableEditEvent UndoableEditListener
- UndoableEditSupport UndoManager UnexpectedException UnicastRemoteObject UnknownError
- UnknownFormatConversionException UnknownFormatFlagsException UnknownGroupException UnknownHostException
- UnknownObjectException UnknownServiceException UnmappableCharacterException UnmarshalException
- UnmodifiableClassException UnmodifiableSetException UnrecoverableEntryException
- UnrecoverableKeyException Unreferenced UnresolvedAddressException UnresolvedPermission
- UnsatisfiedLinkError UnsolicitedNotification UnsolicitedNotificationEvent
- UnsolicitedNotificationListener UnsupportedAddressTypeException UnsupportedAudioFileException
- UnsupportedCallbackException UnsupportedCharsetException UnsupportedClassVersionError
- UnsupportedEncodingException UnsupportedFlavorException UnsupportedLookAndFeelException
- UnsupportedOperationException URI URIException URIResolver URISyntax URISyntaxException URL
- URLClassLoader URLConnection URLDecoder URLEncoder URLStreamHandler URLStreamHandlerFactory
- UTFDataFormatException Util UtilDelegate Utilities UUID Validator ValidatorHandler ValueExp ValueHandler
- ValueHandlerMultiFormat VariableHeightLayoutCache Vector VerifyError VetoableChangeListener
- VetoableChangeListenerProxy VetoableChangeSupport View ViewFactory ViewportLayout ViewportUI
- VirtualMachineError Visibility VMID VoiceStatus Void VolatileImage WeakHashMap WeakReference WebRowSet
- WildcardType Window WindowAdapter WindowConstants WindowEvent WindowFocusListener WindowListener
- WindowStateListener WrappedPlainView WritableByteChannel WritableRaster WritableRenderedImage
- WriteAbortedException Writer X500Principal X500PrivateCredential X509Certificate X509CertSelector
- X509CRL X509CRLEntry X509CRLSelector X509EncodedKeySpec X509ExtendedKeyManager X509Extension
- X509KeyManager X509TrustManager XAConnection XADataSource XAException XAResource Xid XMLConstants
- XMLDecoder XMLEncoder XMLFormatter XMLGregorianCalendar XMLParseException XmlReader XmlWriter XPath
- XPathConstants XPathException XPathExpression XPathExpressionException XPathFactory
- XPathFactoryConfigurationException XPathFunction XPathFunctionException XPathFunctionResolver
- XPathVariableResolver ZipEntry ZipException ZipFile ZipInputStream ZipOutputStream ZoneView
- ]
-
- end
-
-end
-end
+++ /dev/null
-module CodeRay
-module Scanners
-
- class JavaScript < Scanner
-
- include Streamable
-
- register_for :java_script
- file_extension 'js'
-
- # The actual JavaScript keywords.
- KEYWORDS = %w[
- break case catch continue default delete do else
- finally for function if in instanceof new
- return switch throw try typeof var void while with
- ]
- PREDEFINED_CONSTANTS = %w[
- false null true undefined
- ]
-
- MAGIC_VARIABLES = %w[ this arguments ] # arguments was introduced in JavaScript 1.4
-
- KEYWORDS_EXPECTING_VALUE = WordList.new.add %w[
- case delete in instanceof new return throw typeof with
- ]
-
- # Reserved for future use.
- RESERVED_WORDS = %w[
- abstract boolean byte char class debugger double enum export extends
- final float goto implements import int interface long native package
- private protected public short static super synchronized throws transient
- volatile
- ]
-
- IDENT_KIND = WordList.new(:ident).
- add(RESERVED_WORDS, :reserved).
- add(PREDEFINED_CONSTANTS, :pre_constant).
- add(MAGIC_VARIABLES, :local_variable).
- add(KEYWORDS, :keyword)
-
- ESCAPE = / [bfnrtv\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} /x
- UNICODE_ESCAPE = / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} /x
- REGEXP_ESCAPE = / [bBdDsSwW] /x
- STRING_CONTENT_PATTERN = {
- "'" => /[^\\']+/,
- '"' => /[^\\"]+/,
- '/' => /[^\\\/]+/,
- }
- KEY_CHECK_PATTERN = {
- "'" => / [^\\']* (?: \\.? [^\\']* )* '? \s* : /x,
- '"' => / [^\\"]* (?: \\.? [^\\"]* )* "? \s* : /x,
- }
-
- def scan_tokens tokens, options
-
- state = :initial
- string_delimiter = nil
- value_expected = true
- key_expected = false
- function_expected = false
-
- until eos?
-
- kind = nil
- match = nil
-
- case state
-
- when :initial
-
- if match = scan(/ \s+ | \\\n /x)
- value_expected = true if !value_expected && match.index(?\n)
- tokens << [match, :space]
- next
-
- elsif scan(%r! // [^\n\\]* (?: \\. [^\n\\]* )* | /\* (?: .*? \*/ | .* ) !mx)
- value_expected = true
- kind = :comment
-
- elsif check(/\.?\d/)
- key_expected = value_expected = false
- if scan(/0[xX][0-9A-Fa-f]+/)
- kind = :hex
- elsif scan(/(?>0[0-7]+)(?![89.eEfF])/)
- kind = :oct
- elsif scan(/\d+[fF]|\d*\.\d+(?:[eE][+-]?\d+)?[fF]?|\d+[eE][+-]?\d+[fF]?/)
- kind = :float
- elsif scan(/\d+/)
- kind = :integer
- end
-
- elsif value_expected && match = scan(/<([[:alpha:]]\w*) (?: [^\/>]*\/> | .*?<\/\1>)/xim)
- # FIXME: scan over nested tags
- xml_scanner.tokenize match
- value_expected = false
- next
-
- elsif match = scan(/ [-+*=<>?:;,!&^|(\[{~%]+ | \.(?!\d) /x)
- value_expected = true
- last_operator = match[-1]
- key_expected = (last_operator == ?{) || (last_operator == ?,)
- function_expected = false
- kind = :operator
-
- elsif scan(/ [)\]}]+ /x)
- function_expected = key_expected = value_expected = false
- kind = :operator
-
- elsif match = scan(/ [$a-zA-Z_][A-Za-z_0-9$]* /x)
- kind = IDENT_KIND[match]
- value_expected = (kind == :keyword) && KEYWORDS_EXPECTING_VALUE[match]
- # TODO: labels
- if kind == :ident
- if match.index(?$) # $ allowed inside an identifier
- kind = :predefined
- elsif function_expected
- kind = :function
- elsif check(/\s*[=:]\s*function\b/)
- kind = :function
- elsif key_expected && check(/\s*:/)
- kind = :key
- end
- end
- function_expected = (kind == :keyword) && (match == 'function')
- key_expected = false
-
- elsif match = scan(/["']/)
- if key_expected && check(KEY_CHECK_PATTERN[match])
- state = :key
- else
- state = :string
- end
- tokens << [:open, state]
- string_delimiter = match
- kind = :delimiter
-
- elsif value_expected && (match = scan(/\/(?=\S)/))
- tokens << [:open, :regexp]
- state = :regexp
- string_delimiter = '/'
- kind = :delimiter
-
- elsif scan(/ \/ /x)
- value_expected = true
- key_expected = false
- kind = :operator
-
- else
- getch
- kind = :error
-
- end
-
- when :string, :regexp, :key
- if scan(STRING_CONTENT_PATTERN[string_delimiter])
- kind = :content
- elsif match = scan(/["'\/]/)
- tokens << [match, :delimiter]
- if state == :regexp
- modifiers = scan(/[gim]+/)
- tokens << [modifiers, :modifier] if modifiers && !modifiers.empty?
- end
- tokens << [:close, state]
- string_delimiter = nil
- key_expected = value_expected = false
- state = :initial
- next
- elsif state != :regexp && (match = scan(/ \\ (?: #{ESCAPE} | #{UNICODE_ESCAPE} ) /mox))
- if string_delimiter == "'" && !(match == "\\\\" || match == "\\'")
- kind = :content
- else
- kind = :char
- end
- elsif state == :regexp && scan(/ \\ (?: #{ESCAPE} | #{REGEXP_ESCAPE} | #{UNICODE_ESCAPE} ) /mox)
- kind = :char
- elsif scan(/\\./m)
- kind = :content
- elsif scan(/ \\ | $ /x)
- tokens << [:close, state]
- kind = :error
- key_expected = value_expected = false
- state = :initial
- else
- raise_inspect "else case \" reached; %p not handled." % peek(1), tokens
- end
-
- else
- raise_inspect 'Unknown state', tokens
-
- end
-
- match ||= matched
- if $CODERAY_DEBUG and not kind
- raise_inspect 'Error token %p in line %d' %
- [[match, kind], line], tokens
- end
- raise_inspect 'Empty token', tokens unless match
-
- tokens << [match, kind]
-
- end
-
- if [:string, :regexp].include? state
- tokens << [:close, state]
- end
-
- tokens
- end
-
- protected
-
- def reset_instance
- super
- @xml_scanner.reset if defined? @xml_scanner
- end
-
- def xml_scanner
- @xml_scanner ||= CodeRay.scanner :xml, :tokens => @tokens, :keep_tokens => true, :keep_state => true
- end
-
- end
-
-end
-end
+++ /dev/null
-module CodeRay
-module Scanners
-
- class JSON < Scanner
-
- include Streamable
-
- register_for :json
- file_extension 'json'
-
- KINDS_NOT_LOC = [
- :float, :char, :content, :delimiter,
- :error, :integer, :operator, :value,
- ]
-
- CONSTANTS = %w( true false null )
- IDENT_KIND = WordList.new(:key).add(CONSTANTS, :value)
-
- ESCAPE = / [bfnrt\\"\/] /x
- UNICODE_ESCAPE = / u[a-fA-F0-9]{4} /x
-
- def scan_tokens tokens, options
-
- state = :initial
- stack = []
- string_delimiter = nil
- key_expected = false
-
- until eos?
-
- kind = nil
- match = nil
-
- case state
-
- when :initial
- if match = scan(/ \s+ | \\\n /x)
- tokens << [match, :space]
- next
- elsif match = scan(/ [:,\[{\]}] /x)
- kind = :operator
- case match
- when '{' then stack << :object; key_expected = true
- when '[' then stack << :array
- when ':' then key_expected = false
- when ',' then key_expected = true if stack.last == :object
- when '}', ']' then stack.pop # no error recovery, but works for valid JSON
- end
- elsif match = scan(/ true | false | null /x)
- kind = IDENT_KIND[match]
- elsif match = scan(/-?(?:0|[1-9]\d*)/)
- kind = :integer
- if scan(/\.\d+(?:[eE][-+]?\d+)?|[eE][-+]?\d+/)
- match << matched
- kind = :float
- end
- elsif match = scan(/"/)
- state = key_expected ? :key : :string
- tokens << [:open, state]
- kind = :delimiter
- else
- getch
- kind = :error
- end
-
- when :string, :key
- if scan(/[^\\"]+/)
- kind = :content
- elsif scan(/"/)
- tokens << ['"', :delimiter]
- tokens << [:close, state]
- state = :initial
- next
- elsif scan(/ \\ (?: #{ESCAPE} | #{UNICODE_ESCAPE} ) /mox)
- kind = :char
- elsif scan(/\\./m)
- kind = :content
- elsif scan(/ \\ | $ /x)
- tokens << [:close, :delimiter]
- kind = :error
- state = :initial
- else
- raise_inspect "else case \" reached; %p not handled." % peek(1), tokens
- end
-
- else
- raise_inspect 'Unknown state', tokens
-
- end
-
- match ||= matched
- if $CODERAY_DEBUG and not kind
- raise_inspect 'Error token %p in line %d' %
- [[match, kind], line], tokens
- end
- raise_inspect 'Empty token', tokens unless match
-
- tokens << [match, kind]
-
- end
-
- if [:string, :key].include? state
- tokens << [:close, state]
- end
-
- tokens
- end
-
- end
-
-end
-end
+++ /dev/null
-module CodeRay
-module Scanners
-
- load :html
- load :ruby
-
- # Nitro XHTML Scanner
- class NitroXHTML < Scanner
-
- include Streamable
- register_for :nitro_xhtml
- file_extension :xhtml
- title 'Nitro XHTML'
-
- KINDS_NOT_LOC = HTML::KINDS_NOT_LOC
-
- NITRO_RUBY_BLOCK = /
- <\?r
- (?>
- [^\?]*
- (?> \?(?!>) [^\?]* )*
- )
- (?: \?> )?
- |
- <ruby>
- (?>
- [^<]*
- (?> <(?!\/ruby>) [^<]* )*
- )
- (?: <\/ruby> )?
- |
- <%
- (?>
- [^%]*
- (?> %(?!>) [^%]* )*
- )
- (?: %> )?
- /mx
-
- NITRO_VALUE_BLOCK = /
- \#
- (?:
- \{
- [^{}]*
- (?>
- \{ [^}]* \}
- (?> [^{}]* )
- )*
- \}?
- | \| [^|]* \|?
- | \( [^)]* \)?
- | \[ [^\]]* \]?
- | \\ [^\\]* \\?
- )
- /x
-
- NITRO_ENTITY = /
- % (?: \#\d+ | \w+ ) ;
- /
-
- START_OF_RUBY = /
- (?=[<\#%])
- < (?: \?r | % | ruby> )
- | \# [{(|]
- | % (?: \#\d+ | \w+ ) ;
- /x
-
- CLOSING_PAREN = Hash.new do |h, p|
- h[p] = p
- end.update( {
- '(' => ')',
- '[' => ']',
- '{' => '}',
- } )
-
- private
-
- def setup
- @ruby_scanner = CodeRay.scanner :ruby, :tokens => @tokens, :keep_tokens => true
- @html_scanner = CodeRay.scanner :html, :tokens => @tokens, :keep_tokens => true, :keep_state => true
- end
-
- def reset_instance
- super
- @html_scanner.reset
- end
-
- def scan_tokens tokens, options
-
- until eos?
-
- if (match = scan_until(/(?=#{START_OF_RUBY})/o) || scan_until(/\z/)) and not match.empty?
- @html_scanner.tokenize match
-
- elsif match = scan(/#{NITRO_VALUE_BLOCK}/o)
- start_tag = match[0,2]
- delimiter = CLOSING_PAREN[start_tag[1,1]]
- end_tag = match[-1,1] == delimiter ? delimiter : ''
- tokens << [:open, :inline]
- tokens << [start_tag, :inline_delimiter]
- code = match[start_tag.size .. -1 - end_tag.size]
- @ruby_scanner.tokenize code
- tokens << [end_tag, :inline_delimiter] unless end_tag.empty?
- tokens << [:close, :inline]
-
- elsif match = scan(/#{NITRO_RUBY_BLOCK}/o)
- start_tag = '<?r'
- end_tag = match[-2,2] == '?>' ? '?>' : ''
- tokens << [:open, :inline]
- tokens << [start_tag, :inline_delimiter]
- code = match[start_tag.size .. -(end_tag.size)-1]
- @ruby_scanner.tokenize code
- tokens << [end_tag, :inline_delimiter] unless end_tag.empty?
- tokens << [:close, :inline]
-
- elsif entity = scan(/#{NITRO_ENTITY}/o)
- tokens << [entity, :entity]
-
- elsif scan(/%/)
- tokens << [matched, :error]
-
- else
- raise_inspect 'else-case reached!', tokens
-
- end
-
- end
-
- tokens
-
- end
-
- end
-
-end
-end
+++ /dev/null
-module CodeRay
-module Scanners
-
- load :html
-
- # Original by Stefan Walk.
- class PHP < Scanner
-
- register_for :php
- file_extension 'php'
-
- KINDS_NOT_LOC = HTML::KINDS_NOT_LOC
-
- def setup
- @html_scanner = CodeRay.scanner :html, :tokens => @tokens, :keep_tokens => true, :keep_state => true
- end
-
- def reset_instance
- super
- @html_scanner.reset
- end
-
- module Words
-
- # according to http://www.php.net/manual/en/reserved.keywords.php
- KEYWORDS = %w[
- abstract and array as break case catch class clone const continue declare default do else elseif
- enddeclare endfor endforeach endif endswitch endwhile extends final for foreach function global
- goto if implements interface instanceof namespace new or private protected public static switch
- throw try use var while xor
- cfunction old_function
- ]
-
- TYPES = %w[ int integer float double bool boolean string array object resource ]
-
- LANGUAGE_CONSTRUCTS = %w[
- die echo empty exit eval include include_once isset list
- require require_once return print unset
- ]
-
- CLASSES = %w[ Directory stdClass __PHP_Incomplete_Class exception php_user_filter Closure ]
-
- # according to http://php.net/quickref.php on 2009-04-21;
- # all functions with _ excluded (module functions) and selected additional functions
- BUILTIN_FUNCTIONS = %w[
- abs acos acosh addcslashes addslashes aggregate array arsort ascii2ebcdic asin asinh asort assert atan atan2
- atanh basename bcadd bccomp bcdiv bcmod bcmul bcpow bcpowmod bcscale bcsqrt bcsub bin2hex bindec
- bindtextdomain bzclose bzcompress bzdecompress bzerrno bzerror bzerrstr bzflush bzopen bzread bzwrite
- calculhmac ceil chdir checkdate checkdnsrr chgrp chmod chop chown chr chroot clearstatcache closedir closelog
- compact constant copy cos cosh count crc32 crypt current date dcgettext dcngettext deaggregate decbin dechex
- decoct define defined deg2rad delete dgettext die dirname diskfreespace dl dngettext doubleval each
- ebcdic2ascii echo empty end ereg eregi escapeshellarg escapeshellcmd eval exec exit exp explode expm1 extract
- fclose feof fflush fgetc fgetcsv fgets fgetss file fileatime filectime filegroup fileinode filemtime fileowner
- fileperms filepro filesize filetype floatval flock floor flush fmod fnmatch fopen fpassthru fprintf fputcsv
- fputs fread frenchtojd fscanf fseek fsockopen fstat ftell ftok ftruncate fwrite getallheaders getcwd getdate
- getenv gethostbyaddr gethostbyname gethostbynamel getimagesize getlastmod getmxrr getmygid getmyinode getmypid
- getmyuid getopt getprotobyname getprotobynumber getrandmax getrusage getservbyname getservbyport gettext
- gettimeofday gettype glob gmdate gmmktime gmstrftime gregoriantojd gzclose gzcompress gzdecode gzdeflate
- gzencode gzeof gzfile gzgetc gzgets gzgetss gzinflate gzopen gzpassthru gzputs gzread gzrewind gzseek gztell
- gzuncompress gzwrite hash header hebrev hebrevc hexdec htmlentities htmlspecialchars hypot iconv idate
- implode include intval ip2long iptcembed iptcparse isset
- jddayofweek jdmonthname jdtofrench jdtogregorian jdtojewish jdtojulian jdtounix jewishtojd join jpeg2wbmp
- juliantojd key krsort ksort lcfirst lchgrp lchown levenshtein link linkinfo list localeconv localtime log
- log10 log1p long2ip lstat ltrim mail main max md5 metaphone mhash microtime min mkdir mktime msql natcasesort
- natsort next ngettext nl2br nthmac octdec opendir openlog
- ord overload pack passthru pathinfo pclose pfsockopen phpcredits phpinfo phpversion pi png2wbmp popen pos pow
- prev print printf putenv quotemeta rad2deg rand range rawurldecode rawurlencode readdir readfile readgzfile
- readline readlink realpath recode rename require reset rewind rewinddir rmdir round rsort rtrim scandir
- serialize setcookie setlocale setrawcookie settype sha1 shuffle signeurlpaiement sin sinh sizeof sleep snmpget
- snmpgetnext snmprealwalk snmpset snmpwalk snmpwalkoid sort soundex split spliti sprintf sqrt srand sscanf stat
- strcasecmp strchr strcmp strcoll strcspn strftime stripcslashes stripos stripslashes stristr strlen
- strnatcasecmp strnatcmp strncasecmp strncmp strpbrk strpos strptime strrchr strrev strripos strrpos strspn
- strstr strtok strtolower strtotime strtoupper strtr strval substr symlink syslog system tan tanh tempnam
- textdomain time tmpfile touch trim uasort ucfirst ucwords uksort umask uniqid unixtojd unlink unpack
- unserialize unset urldecode urlencode usleep usort vfprintf virtual vprintf vsprintf wordwrap
- array_change_key_case array_chunk array_combine array_count_values array_diff array_diff_assoc
- array_diff_key array_diff_uassoc array_diff_ukey array_fill array_fill_keys array_filter array_flip
- array_intersect array_intersect_assoc array_intersect_key array_intersect_uassoc array_intersect_ukey
- array_key_exists array_keys array_map array_merge array_merge_recursive array_multisort array_pad
- array_pop array_product array_push array_rand array_reduce array_reverse array_search array_shift
- array_slice array_splice array_sum array_udiff array_udiff_assoc array_udiff_uassoc array_uintersect
- array_uintersect_assoc array_uintersect_uassoc array_unique array_unshift array_values array_walk
- array_walk_recursive
- assert_options base_convert base64_decode base64_encode
- chunk_split class_exists class_implements class_parents
- count_chars debug_backtrace debug_print_backtrace debug_zval_dump
- error_get_last error_log error_reporting extension_loaded
- file_exists file_get_contents file_put_contents load_file
- func_get_arg func_get_args func_num_args function_exists
- get_browser get_called_class get_cfg_var get_class get_class_methods get_class_vars
- get_current_user get_declared_classes get_declared_interfaces get_defined_constants
- get_defined_functions get_defined_vars get_extension_funcs get_headers get_html_translation_table
- get_include_path get_included_files get_loaded_extensions get_magic_quotes_gpc get_magic_quotes_runtime
- get_meta_tags get_object_vars get_parent_class get_required_filesget_resource_type
- gc_collect_cycles gc_disable gc_enable gc_enabled
- halt_compiler headers_list headers_sent highlight_file highlight_string
- html_entity_decode htmlspecialchars_decode
- in_array include_once inclued_get_data
- is_a is_array is_binary is_bool is_buffer is_callable is_dir is_double is_executable is_file is_finite
- is_float is_infinite is_int is_integer is_link is_long is_nan is_null is_numeric is_object is_readable
- is_real is_resource is_scalar is_soap_fault is_string is_subclass_of is_unicode is_uploaded_file
- is_writable is_writeable
- locale_get_default locale_set_default
- number_format override_function parse_str parse_url
- php_check_syntax php_ini_loaded_file php_ini_scanned_files php_logo_guid php_sapi_name
- php_strip_whitespace php_uname
- preg_filter preg_grep preg_last_error preg_match preg_match_all preg_quote preg_replace
- preg_replace_callback preg_split print_r
- require_once register_shutdown_function register_tick_function
- set_error_handler set_exception_handler set_file_buffer set_include_path
- set_magic_quotes_runtime set_time_limit shell_exec
- str_getcsv str_ireplace str_pad str_repeat str_replace str_rot13 str_shuffle str_split str_word_count
- strip_tags substr_compare substr_count substr_replace
- time_nanosleep time_sleep_until
- token_get_all token_name trigger_error
- unregister_tick_function use_soap_error_handler user_error
- utf8_decode utf8_encode var_dump var_export
- version_compare
- zend_logo_guid zend_thread_id zend_version
- create_function call_user_func_array
- posix_access posix_ctermid posix_get_last_error posix_getcwd posix_getegid
- posix_geteuid posix_getgid posix_getgrgid posix_getgrnam posix_getgroups
- posix_getlogin posix_getpgid posix_getpgrp posix_getpid posix_getppid
- posix_getpwnam posix_getpwuid posix_getrlimit posix_getsid posix_getuid
- posix_initgroups posix_isatty posix_kill posix_mkfifo posix_mknod
- posix_setegid posix_seteuid posix_setgid posix_setpgid posix_setsid
- posix_setuid posix_strerror posix_times posix_ttyname posix_uname
- pcntl_alarm pcntl_exec pcntl_fork pcntl_getpriority pcntl_setpriority
- pcntl_signal pcntl_signal_dispatch pcntl_sigprocmask pcntl_sigtimedwait
- pcntl_sigwaitinfo pcntl_wait pcntl_waitpid pcntl_wexitstatus pcntl_wifexited
- pcntl_wifsignaled pcntl_wifstopped pcntl_wstopsig pcntl_wtermsig
- ]
- # TODO: more built-in PHP functions?
-
- EXCEPTIONS = %w[
- E_ERROR E_WARNING E_PARSE E_NOTICE E_CORE_ERROR E_CORE_WARNING E_COMPILE_ERROR E_COMPILE_WARNING
- E_USER_ERROR E_USER_WARNING E_USER_NOTICE E_DEPRECATED E_USER_DEPRECATED E_ALL E_STRICT
- ]
-
- CONSTANTS = %w[
- null true false self parent
- __LINE__ __DIR__ __FILE__ __LINE__
- __CLASS__ __NAMESPACE__ __METHOD__ __FUNCTION__
- PHP_VERSION PHP_MAJOR_VERSION PHP_MINOR_VERSION PHP_RELEASE_VERSION PHP_VERSION_ID PHP_EXTRA_VERSION PHP_ZTS
- PHP_DEBUG PHP_MAXPATHLEN PHP_OS PHP_SAPI PHP_EOL PHP_INT_MAX PHP_INT_SIZE DEFAULT_INCLUDE_PATH
- PEAR_INSTALL_DIR PEAR_EXTENSION_DIR PHP_EXTENSION_DIR PHP_PREFIX PHP_BINDIR PHP_LIBDIR PHP_DATADIR
- PHP_SYSCONFDIR PHP_LOCALSTATEDIR PHP_CONFIG_FILE_PATH PHP_CONFIG_FILE_SCAN_DIR PHP_SHLIB_SUFFIX
- PHP_OUTPUT_HANDLER_START PHP_OUTPUT_HANDLER_CONT PHP_OUTPUT_HANDLER_END
- __COMPILER_HALT_OFFSET__
- EXTR_OVERWRITE EXTR_SKIP EXTR_PREFIX_SAME EXTR_PREFIX_ALL EXTR_PREFIX_INVALID EXTR_PREFIX_IF_EXISTS
- EXTR_IF_EXISTS SORT_ASC SORT_DESC SORT_REGULAR SORT_NUMERIC SORT_STRING CASE_LOWER CASE_UPPER COUNT_NORMAL
- COUNT_RECURSIVE ASSERT_ACTIVE ASSERT_CALLBACK ASSERT_BAIL ASSERT_WARNING ASSERT_QUIET_EVAL CONNECTION_ABORTED
- CONNECTION_NORMAL CONNECTION_TIMEOUT INI_USER INI_PERDIR INI_SYSTEM INI_ALL M_E M_LOG2E M_LOG10E M_LN2 M_LN10
- M_PI M_PI_2 M_PI_4 M_1_PI M_2_PI M_2_SQRTPI M_SQRT2 M_SQRT1_2 CRYPT_SALT_LENGTH CRYPT_STD_DES CRYPT_EXT_DES
- CRYPT_MD5 CRYPT_BLOWFISH DIRECTORY_SEPARATOR SEEK_SET SEEK_CUR SEEK_END LOCK_SH LOCK_EX LOCK_UN LOCK_NB
- HTML_SPECIALCHARS HTML_ENTITIES ENT_COMPAT ENT_QUOTES ENT_NOQUOTES INFO_GENERAL INFO_CREDITS
- INFO_CONFIGURATION INFO_MODULES INFO_ENVIRONMENT INFO_VARIABLES INFO_LICENSE INFO_ALL CREDITS_GROUP
- CREDITS_GENERAL CREDITS_SAPI CREDITS_MODULES CREDITS_DOCS CREDITS_FULLPAGE CREDITS_QA CREDITS_ALL STR_PAD_LEFT
- STR_PAD_RIGHT STR_PAD_BOTH PATHINFO_DIRNAME PATHINFO_BASENAME PATHINFO_EXTENSION PATH_SEPARATOR CHAR_MAX
- LC_CTYPE LC_NUMERIC LC_TIME LC_COLLATE LC_MONETARY LC_ALL LC_MESSAGES ABDAY_1 ABDAY_2 ABDAY_3 ABDAY_4 ABDAY_5
- ABDAY_6 ABDAY_7 DAY_1 DAY_2 DAY_3 DAY_4 DAY_5 DAY_6 DAY_7 ABMON_1 ABMON_2 ABMON_3 ABMON_4 ABMON_5 ABMON_6
- ABMON_7 ABMON_8 ABMON_9 ABMON_10 ABMON_11 ABMON_12 MON_1 MON_2 MON_3 MON_4 MON_5 MON_6 MON_7 MON_8 MON_9
- MON_10 MON_11 MON_12 AM_STR PM_STR D_T_FMT D_FMT T_FMT T_FMT_AMPM ERA ERA_YEAR ERA_D_T_FMT ERA_D_FMT ERA_T_FMT
- ALT_DIGITS INT_CURR_SYMBOL CURRENCY_SYMBOL CRNCYSTR MON_DECIMAL_POINT MON_THOUSANDS_SEP MON_GROUPING
- POSITIVE_SIGN NEGATIVE_SIGN INT_FRAC_DIGITS FRAC_DIGITS P_CS_PRECEDES P_SEP_BY_SPACE N_CS_PRECEDES
- N_SEP_BY_SPACE P_SIGN_POSN N_SIGN_POSN DECIMAL_POINT RADIXCHAR THOUSANDS_SEP THOUSEP GROUPING YESEXPR NOEXPR
- YESSTR NOSTR CODESET LOG_EMERG LOG_ALERT LOG_CRIT LOG_ERR LOG_WARNING LOG_NOTICE LOG_INFO LOG_DEBUG LOG_KERN
- LOG_USER LOG_MAIL LOG_DAEMON LOG_AUTH LOG_SYSLOG LOG_LPR LOG_NEWS LOG_UUCP LOG_CRON LOG_AUTHPRIV LOG_LOCAL0
- LOG_LOCAL1 LOG_LOCAL2 LOG_LOCAL3 LOG_LOCAL4 LOG_LOCAL5 LOG_LOCAL6 LOG_LOCAL7 LOG_PID LOG_CONS LOG_ODELAY
- LOG_NDELAY LOG_NOWAIT LOG_PERROR
- ]
-
- PREDEFINED = %w[
- $GLOBALS $_SERVER $_GET $_POST $_FILES $_REQUEST $_SESSION $_ENV
- $_COOKIE $php_errormsg $HTTP_RAW_POST_DATA $http_response_header
- $argc $argv
- ]
-
- IDENT_KIND = CaseIgnoringWordList.new(:ident).
- add(KEYWORDS, :reserved).
- add(TYPES, :pre_type).
- add(LANGUAGE_CONSTRUCTS, :reserved).
- add(BUILTIN_FUNCTIONS, :predefined).
- add(CLASSES, :pre_constant).
- add(EXCEPTIONS, :exception).
- add(CONSTANTS, :pre_constant)
-
- VARIABLE_KIND = WordList.new(:local_variable).
- add(PREDEFINED, :predefined)
- end
-
- module RE
-
- PHP_START = /
- <script\s+[^>]*?language\s*=\s*"php"[^>]*?> |
- <script\s+[^>]*?language\s*=\s*'php'[^>]*?> |
- <\?php\d? |
- <\?(?!xml)
- /xi
-
- PHP_END = %r!
- </script> |
- \?>
- !xi
-
- HTML_INDICATOR = /<!DOCTYPE html|<(?:html|body|div|p)[> ]/i
-
- IDENTIFIER = /[a-z_\x7f-\xFF][a-z0-9_\x7f-\xFF]*/i
- VARIABLE = /\$#{IDENTIFIER}/
-
- OPERATOR = /
- \.(?!\d)=? | # dot that is not decimal point, string concatenation
- && | \|\| | # logic
- :: | -> | => | # scope, member, dictionary
- \\(?!\n) | # namespace
- \+\+ | -- | # increment, decrement
- [,;?:()\[\]{}] | # simple delimiters
- [-+*\/%&|^]=? | # ordinary math, binary logic, assignment shortcuts
- [~$] | # whatever
- =& | # reference assignment
- [=!]=?=? | <> | # comparison and assignment
- <<=? | >>=? | [<>]=? # comparison and shift
- /x
-
- end
-
- def scan_tokens tokens, options
-
- if check(RE::PHP_START) || # starts with <?
- (match?(/\s*<\S/) && exist?(RE::PHP_START)) || # starts with tag and contains <?
- exist?(RE::HTML_INDICATOR) ||
- check(/.{1,100}#{RE::PHP_START}/om) # PHP start after max 100 chars
- # is HTML with embedded PHP, so start with HTML
- states = [:initial]
- else
- # is just PHP, so start with PHP surrounded by HTML
- states = [:initial, :php]
- end
-
- label_expected = true
- case_expected = false
-
- heredoc_delimiter = nil
- delimiter = nil
- modifier = nil
-
- until eos?
-
- match = nil
- kind = nil
-
- case states.last
-
- when :initial # HTML
- if scan RE::PHP_START
- kind = :inline_delimiter
- label_expected = true
- states << :php
- else
- match = scan_until(/(?=#{RE::PHP_START})/o) || scan_until(/\z/)
- @html_scanner.tokenize match unless match.empty?
- next
- end
-
- when :php
- if match = scan(/\s+/)
- tokens << [match, :space]
- next
-
- elsif scan(%r! (?m: \/\* (?: .*? \*\/ | .* ) ) | (?://|\#) .*? (?=#{RE::PHP_END}|$) !xo)
- kind = :comment
-
- elsif match = scan(RE::IDENTIFIER)
- kind = Words::IDENT_KIND[match]
- if kind == :ident && label_expected && check(/:(?!:)/)
- kind = :label
- label_expected = true
- else
- label_expected = false
- if kind == :ident && match =~ /^[A-Z]/
- kind = :constant
- elsif kind == :reserved
- case match
- when 'class'
- states << :class_expected
- when 'function'
- states << :function_expected
- when 'case', 'default'
- case_expected = true
- end
- elsif match == 'b' && check(/['"]/) # binary string literal
- modifier = match
- next
- end
- end
-
- elsif scan(/(?:\d+\.\d*|\d*\.\d+)(?:e[-+]?\d+)?|\d+e[-+]?\d+/i)
- label_expected = false
- kind = :float
-
- elsif scan(/0x[0-9a-fA-F]+/)
- label_expected = false
- kind = :hex
-
- elsif scan(/\d+/)
- label_expected = false
- kind = :integer
-
- elsif scan(/'/)
- tokens << [:open, :string]
- if modifier
- tokens << [modifier, :modifier]
- modifier = nil
- end
- kind = :delimiter
- states.push :sqstring
-
- elsif match = scan(/["`]/)
- tokens << [:open, :string]
- if modifier
- tokens << [modifier, :modifier]
- modifier = nil
- end
- delimiter = match
- kind = :delimiter
- states.push :dqstring
-
- elsif match = scan(RE::VARIABLE)
- label_expected = false
- kind = Words::VARIABLE_KIND[match]
-
- elsif scan(/\{/)
- kind = :operator
- label_expected = true
- states.push :php
-
- elsif scan(/\}/)
- if states.size == 1
- kind = :error
- else
- states.pop
- if states.last.is_a?(::Array)
- delimiter = states.last[1]
- states[-1] = states.last[0]
- tokens << [matched, :delimiter]
- tokens << [:close, :inline]
- next
- else
- kind = :operator
- label_expected = true
- end
- end
-
- elsif scan(/@/)
- label_expected = false
- kind = :exception
-
- elsif scan RE::PHP_END
- kind = :inline_delimiter
- states = [:initial]
-
- elsif match = scan(/<<<(?:(#{RE::IDENTIFIER})|"(#{RE::IDENTIFIER})"|'(#{RE::IDENTIFIER})')/o)
- tokens << [:open, :string]
- warn 'heredoc in heredoc?' if heredoc_delimiter
- heredoc_delimiter = Regexp.escape(self[1] || self[2] || self[3])
- kind = :delimiter
- states.push self[3] ? :sqstring : :dqstring
- heredoc_delimiter = /#{heredoc_delimiter}(?=;?$)/
-
- elsif match = scan(/#{RE::OPERATOR}/o)
- label_expected = match == ';'
- if case_expected
- label_expected = true if match == ':'
- case_expected = false
- end
- kind = :operator
-
- else
- getch
- kind = :error
-
- end
-
- when :sqstring
- if scan(heredoc_delimiter ? /[^\\\n]+/ : /[^'\\]+/)
- kind = :content
- elsif !heredoc_delimiter && scan(/'/)
- tokens << [matched, :delimiter]
- tokens << [:close, :string]
- delimiter = nil
- label_expected = false
- states.pop
- next
- elsif heredoc_delimiter && match = scan(/\n/)
- kind = :content
- if scan heredoc_delimiter
- tokens << ["\n", :content]
- tokens << [matched, :delimiter]
- tokens << [:close, :string]
- heredoc_delimiter = nil
- label_expected = false
- states.pop
- next
- end
- elsif scan(heredoc_delimiter ? /\\\\/ : /\\[\\'\n]/)
- kind = :char
- elsif scan(/\\./m)
- kind = :content
- elsif scan(/\\/)
- kind = :error
- end
-
- when :dqstring
- if scan(heredoc_delimiter ? /[^${\\\n]+/ : (delimiter == '"' ? /[^"${\\]+/ : /[^`${\\]+/))
- kind = :content
- elsif !heredoc_delimiter && scan(delimiter == '"' ? /"/ : /`/)
- tokens << [matched, :delimiter]
- tokens << [:close, :string]
- delimiter = nil
- label_expected = false
- states.pop
- next
- elsif heredoc_delimiter && match = scan(/\n/)
- kind = :content
- if scan heredoc_delimiter
- tokens << ["\n", :content]
- tokens << [matched, :delimiter]
- tokens << [:close, :string]
- heredoc_delimiter = nil
- label_expected = false
- states.pop
- next
- end
- elsif scan(/\\(?:x[0-9A-Fa-f]{1,2}|[0-7]{1,3})/)
- kind = :char
- elsif scan(heredoc_delimiter ? /\\[nrtvf\\$]/ : (delimiter == '"' ? /\\[nrtvf\\$"]/ : /\\[nrtvf\\$`]/))
- kind = :char
- elsif scan(/\\./m)
- kind = :content
- elsif scan(/\\/)
- kind = :error
- elsif match = scan(/#{RE::VARIABLE}/o)
- kind = :local_variable
- if check(/\[#{RE::IDENTIFIER}\]/o)
- tokens << [:open, :inline]
- tokens << [match, :local_variable]
- tokens << [scan(/\[/), :operator]
- tokens << [scan(/#{RE::IDENTIFIER}/o), :ident]
- tokens << [scan(/\]/), :operator]
- tokens << [:close, :inline]
- next
- elsif check(/\[/)
- match << scan(/\[['"]?#{RE::IDENTIFIER}?['"]?\]?/o)
- kind = :error
- elsif check(/->#{RE::IDENTIFIER}/o)
- tokens << [:open, :inline]
- tokens << [match, :local_variable]
- tokens << [scan(/->/), :operator]
- tokens << [scan(/#{RE::IDENTIFIER}/o), :ident]
- tokens << [:close, :inline]
- next
- elsif check(/->/)
- match << scan(/->/)
- kind = :error
- end
- elsif match = scan(/\{/)
- if check(/\$/)
- kind = :delimiter
- states[-1] = [states.last, delimiter]
- delimiter = nil
- states.push :php
- tokens << [:open, :inline]
- else
- kind = :string
- end
- elsif scan(/\$\{#{RE::IDENTIFIER}\}/o)
- kind = :local_variable
- elsif scan(/\$/)
- kind = :content
- end
-
- when :class_expected
- if scan(/\s+/)
- kind = :space
- elsif match = scan(/#{RE::IDENTIFIER}/o)
- kind = :class
- states.pop
- else
- states.pop
- next
- end
-
- when :function_expected
- if scan(/\s+/)
- kind = :space
- elsif scan(/&/)
- kind = :operator
- elsif match = scan(/#{RE::IDENTIFIER}/o)
- kind = :function
- states.pop
- else
- states.pop
- next
- end
-
- else
- raise_inspect 'Unknown state!', tokens, states
- end
-
- match ||= matched
- if $CODERAY_DEBUG and not kind
- raise_inspect 'Error token %p in line %d' %
- [[match, kind], line], tokens, states
- end
- raise_inspect 'Empty token', tokens, states unless match
-
- tokens << [match, kind]
-
- end
-
- tokens
- end
-
- end
-
-end
-end
+++ /dev/null
-module CodeRay
-module Scanners
-
- class Plaintext < Scanner
-
- register_for :plaintext, :plain
- title 'Plain text'
-
- include Streamable
-
- KINDS_NOT_LOC = [:plain]
-
- def scan_tokens tokens, options
- text = (scan_until(/\z/) || '')
- tokens << [text, :plain]
- end
-
- end
-
-end
-end
+++ /dev/null
-module CodeRay
-module Scanners
-
- # Bases on pygments' PythonLexer, see
- # http://dev.pocoo.org/projects/pygments/browser/pygments/lexers/agile.py.
- class Python < Scanner
-
- include Streamable
-
- register_for :python
- file_extension 'py'
-
- KEYWORDS = [
- 'and', 'as', 'assert', 'break', 'class', 'continue', 'def',
- 'del', 'elif', 'else', 'except', 'finally', 'for',
- 'from', 'global', 'if', 'import', 'in', 'is', 'lambda', 'not',
- 'or', 'pass', 'raise', 'return', 'try', 'while', 'with', 'yield',
- 'nonlocal', # new in Python 3
- ]
-
- OLD_KEYWORDS = [
- 'exec', 'print', # gone in Python 3
- ]
-
- PREDEFINED_METHODS_AND_TYPES = %w[
- __import__ abs all any apply basestring bin bool buffer
- bytearray bytes callable chr classmethod cmp coerce compile
- complex delattr dict dir divmod enumerate eval execfile exit
- file filter float frozenset getattr globals hasattr hash hex id
- input int intern isinstance issubclass iter len list locals
- long map max min next object oct open ord pow property range
- raw_input reduce reload repr reversed round set setattr slice
- sorted staticmethod str sum super tuple type unichr unicode
- vars xrange zip
- ]
-
- PREDEFINED_EXCEPTIONS = %w[
- ArithmeticError AssertionError AttributeError
- BaseException DeprecationWarning EOFError EnvironmentError
- Exception FloatingPointError FutureWarning GeneratorExit IOError
- ImportError ImportWarning IndentationError IndexError KeyError
- KeyboardInterrupt LookupError MemoryError NameError
- NotImplemented NotImplementedError OSError OverflowError
- OverflowWarning PendingDeprecationWarning ReferenceError
- RuntimeError RuntimeWarning StandardError StopIteration
- SyntaxError SyntaxWarning SystemError SystemExit TabError
- TypeError UnboundLocalError UnicodeDecodeError
- UnicodeEncodeError UnicodeError UnicodeTranslateError
- UnicodeWarning UserWarning ValueError Warning ZeroDivisionError
- ]
-
- PREDEFINED_VARIABLES_AND_CONSTANTS = [
- 'False', 'True', 'None', # "keywords" since Python 3
- 'self', 'Ellipsis', 'NotImplemented',
- ]
-
- IDENT_KIND = WordList.new(:ident).
- add(KEYWORDS, :keyword).
- add(OLD_KEYWORDS, :old_keyword).
- add(PREDEFINED_METHODS_AND_TYPES, :predefined).
- add(PREDEFINED_VARIABLES_AND_CONSTANTS, :pre_constant).
- add(PREDEFINED_EXCEPTIONS, :exception)
-
- NAME = / [^\W\d] \w* /x
- ESCAPE = / [abfnrtv\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} /x
- UNICODE_ESCAPE = / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} | N\{[-\w ]+\} /x
-
- OPERATOR = /
- \.\.\. | # ellipsis
- \.(?!\d) | # dot but not decimal point
- [,;:()\[\]{}] | # simple delimiters
- \/\/=? | \*\*=? | # special math
- [-+*\/%&|^]=? | # ordinary math and binary logic
- [~`] | # binary complement and inspection
- <<=? | >>=? | [<>=]=? | != # comparison and assignment
- /x
-
- STRING_DELIMITER_REGEXP = Hash.new do |h, delimiter|
- h[delimiter] = Regexp.union delimiter
- end
-
- STRING_CONTENT_REGEXP = Hash.new do |h, delimiter|
- h[delimiter] = / [^\\\n]+? (?= \\ | $ | #{Regexp.escape(delimiter)} ) /x
- end
-
- DEF_NEW_STATE = WordList.new(:initial).
- add(%w(def), :def_expected).
- add(%w(import from), :include_expected).
- add(%w(class), :class_expected)
-
- DESCRIPTOR = /
- #{NAME}
- (?: \. #{NAME} )*
- | \*
- /x
-
- def scan_tokens tokens, options
-
- state = :initial
- string_delimiter = nil
- string_raw = false
- import_clause = class_name_follows = last_token_dot = false
- unicode = string.respond_to?(:encoding) && string.encoding.name == 'UTF-8'
- from_import_state = []
-
- until eos?
-
- kind = nil
- match = nil
-
- if state == :string
- if scan(STRING_DELIMITER_REGEXP[string_delimiter])
- tokens << [matched, :delimiter]
- tokens << [:close, :string]
- state = :initial
- next
- elsif string_delimiter.size == 3 && scan(/\n/)
- kind = :content
- elsif scan(STRING_CONTENT_REGEXP[string_delimiter])
- kind = :content
- elsif !string_raw && scan(/ \\ #{ESCAPE} /ox)
- kind = :char
- elsif scan(/ \\ #{UNICODE_ESCAPE} /ox)
- kind = :char
- elsif scan(/ \\ . /x)
- kind = :content
- elsif scan(/ \\ | $ /x)
- tokens << [:close, :string]
- kind = :error
- state = :initial
- else
- raise_inspect "else case \" reached; %p not handled." % peek(1), tokens, state
- end
-
- elsif match = scan(/ [ \t]+ | \\\n /x)
- tokens << [match, :space]
- next
-
- elsif match = scan(/\n/)
- tokens << [match, :space]
- state = :initial if state == :include_expected
- next
-
- elsif match = scan(/ \# [^\n]* /mx)
- tokens << [match, :comment]
- next
-
- elsif state == :initial
-
- if scan(/#{OPERATOR}/o)
- kind = :operator
-
- elsif match = scan(/(u?r?|b)?("""|"|'''|')/i)
- tokens << [:open, :string]
- string_delimiter = self[2]
- string_raw = false
- modifiers = self[1]
- unless modifiers.empty?
- string_raw = !!modifiers.index(?r)
- tokens << [modifiers, :modifier]
- match = string_delimiter
- end
- state = :string
- kind = :delimiter
-
- # TODO: backticks
-
- elsif match = scan(unicode ? /#{NAME}/uo : /#{NAME}/o)
- kind = IDENT_KIND[match]
- # TODO: keyword arguments
- kind = :ident if last_token_dot
- if kind == :old_keyword
- kind = check(/\(/) ? :ident : :keyword
- elsif kind == :predefined && check(/ *=/)
- kind = :ident
- elsif kind == :keyword
- state = DEF_NEW_STATE[match]
- from_import_state << match.to_sym if state == :include_expected
- end
-
- elsif scan(/@[a-zA-Z0-9_.]+[lL]?/)
- kind = :decorator
-
- elsif scan(/0[xX][0-9A-Fa-f]+[lL]?/)
- kind = :hex
-
- elsif scan(/0[bB][01]+[lL]?/)
- kind = :bin
-
- elsif match = scan(/(?:\d*\.\d+|\d+\.\d*)(?:[eE][+-]?\d+)?|\d+[eE][+-]?\d+/)
- kind = :float
- if scan(/[jJ]/)
- match << matched
- kind = :imaginary
- end
-
- elsif scan(/0[oO][0-7]+|0[0-7]+(?![89.eE])[lL]?/)
- kind = :oct
-
- elsif match = scan(/\d+([lL])?/)
- kind = :integer
- if self[1] == nil && scan(/[jJ]/)
- match << matched
- kind = :imaginary
- end
-
- else
- getch
- kind = :error
-
- end
-
- elsif state == :def_expected
- state = :initial
- if match = scan(unicode ? /#{NAME}/uo : /#{NAME}/o)
- kind = :method
- else
- next
- end
-
- elsif state == :class_expected
- state = :initial
- if match = scan(unicode ? /#{NAME}/uo : /#{NAME}/o)
- kind = :class
- else
- next
- end
-
- elsif state == :include_expected
- if match = scan(unicode ? /#{DESCRIPTOR}/uo : /#{DESCRIPTOR}/o)
- kind = :include
- if match == 'as'
- kind = :keyword
- from_import_state << :as
- elsif from_import_state.first == :from && match == 'import'
- kind = :keyword
- from_import_state << :import
- elsif from_import_state.last == :as
- # kind = match[0,1][unicode ? /[[:upper:]]/u : /[[:upper:]]/] ? :class : :method
- kind = :ident
- from_import_state.pop
- elsif IDENT_KIND[match] == :keyword
- unscan
- match = nil
- state = :initial
- next
- end
- elsif match = scan(/,/)
- from_import_state.pop if from_import_state.last == :as
- kind = :operator
- else
- from_import_state = []
- state = :initial
- next
- end
-
- else
- raise_inspect 'Unknown state', tokens, state
-
- end
-
- match ||= matched
- if $CODERAY_DEBUG and not kind
- raise_inspect 'Error token %p in line %d' %
- [[match, kind], line], tokens, state
- end
- raise_inspect 'Empty token', tokens, state unless match
-
- last_token_dot = match == '.'
-
- tokens << [match, kind]
-
- end
-
- if state == :string
- tokens << [:close, :string]
- end
-
- tokens
- end
-
- end
-
-end
-end
+++ /dev/null
-module CodeRay
-module Scanners
-
- load :html
- load :ruby
-
- # RHTML Scanner
- class RHTML < Scanner
-
- include Streamable
- register_for :rhtml
- title 'HTML ERB Template'
-
- KINDS_NOT_LOC = HTML::KINDS_NOT_LOC
-
- ERB_RUBY_BLOCK = /
- <%(?!%)[=-]?
- (?>
- [^\-%]* # normal*
- (?> # special
- (?: %(?!>) | -(?!%>) )
- [^\-%]* # normal*
- )*
- )
- (?: -?%> )?
- /x
-
- START_OF_ERB = /
- <%(?!%)
- /x
-
- private
-
- def setup
- @ruby_scanner = CodeRay.scanner :ruby, :tokens => @tokens, :keep_tokens => true
- @html_scanner = CodeRay.scanner :html, :tokens => @tokens, :keep_tokens => true, :keep_state => true
- end
-
- def reset_instance
- super
- @html_scanner.reset
- end
-
- def scan_tokens tokens, options
-
- until eos?
-
- if (match = scan_until(/(?=#{START_OF_ERB})/o) || scan_until(/\z/)) and not match.empty?
- @html_scanner.tokenize match
-
- elsif match = scan(/#{ERB_RUBY_BLOCK}/o)
- start_tag = match[/\A<%[-=]?/]
- end_tag = match[/-?%?>?\z/]
- tokens << [:open, :inline]
- tokens << [start_tag, :inline_delimiter]
- code = match[start_tag.size .. -1 - end_tag.size]
- @ruby_scanner.tokenize code
- tokens << [end_tag, :inline_delimiter] unless end_tag.empty?
- tokens << [:close, :inline]
-
- else
- raise_inspect 'else-case reached!', tokens
- end
-
- end
-
- tokens
-
- end
-
- end
-
-end
-end
+++ /dev/null
-module CodeRay
-module Scanners
-
- # This scanner is really complex, since Ruby _is_ a complex language!
- #
- # It tries to highlight 100% of all common code,
- # and 90% of strange codes.
- #
- # It is optimized for HTML highlighting, and is not very useful for
- # parsing or pretty printing.
- #
- # For now, I think it's better than the scanners in VIM or Syntax, or
- # any highlighter I was able to find, except Caleb's RubyLexer.
- #
- # I hope it's also better than the rdoc/irb lexer.
- class Ruby < Scanner
-
- include Streamable
-
- register_for :ruby
- file_extension 'rb'
-
- helper :patterns
-
- if not defined? EncodingError
- EncodingError = Class.new Exception
- end
-
- private
- def scan_tokens tokens, options
- last_token_dot = false
- value_expected = true
- heredocs = nil
- last_state = nil
- state = :initial
- depth = nil
- inline_block_stack = []
- unicode = string.respond_to?(:encoding) && string.encoding.name == 'UTF-8'
-
- patterns = Patterns # avoid constant lookup
-
- until eos?
- match = nil
- kind = nil
-
- if state.instance_of? patterns::StringState
-# {{{
- match = scan_until(state.pattern) || scan_until(/\z/)
- tokens << [match, :content] unless match.empty?
- break if eos?
-
- if state.heredoc and self[1] # end of heredoc
- match = getch.to_s
- match << scan_until(/$/) unless eos?
- tokens << [match, :delimiter]
- tokens << [:close, state.type]
- state = state.next_state
- next
- end
-
- case match = getch
-
- when state.delim
- if state.paren
- state.paren_depth -= 1
- if state.paren_depth > 0
- tokens << [match, :nesting_delimiter]
- next
- end
- end
- tokens << [match, :delimiter]
- if state.type == :regexp and not eos?
- modifiers = scan(/#{patterns::REGEXP_MODIFIERS}/ox)
- tokens << [modifiers, :modifier] unless modifiers.empty?
- end
- tokens << [:close, state.type]
- value_expected = false
- state = state.next_state
-
- when '\\'
- if state.interpreted
- if esc = scan(/ #{patterns::ESCAPE} /ox)
- tokens << [match + esc, :char]
- else
- tokens << [match, :error]
- end
- else
- case m = getch
- when state.delim, '\\'
- tokens << [match + m, :char]
- when nil
- tokens << [match, :error]
- else
- tokens << [match + m, :content]
- end
- end
-
- when '#'
- case peek(1)
- when '{'
- inline_block_stack << [state, depth, heredocs]
- value_expected = true
- state = :initial
- depth = 1
- tokens << [:open, :inline]
- tokens << [match + getch, :inline_delimiter]
- when '$', '@'
- tokens << [match, :escape]
- last_state = state # scan one token as normal code, then return here
- state = :initial
- else
- raise_inspect 'else-case # reached; #%p not handled' % peek(1), tokens
- end
-
- when state.paren
- state.paren_depth += 1
- tokens << [match, :nesting_delimiter]
-
- when /#{patterns::REGEXP_SYMBOLS}/ox
- tokens << [match, :function]
-
- else
- raise_inspect 'else-case " reached; %p not handled, state = %p' % [match, state], tokens
-
- end
- next
-# }}}
- else
-# {{{
- if match = scan(/[ \t\f]+/)
- kind = :space
- match << scan(/\s*/) unless eos? || heredocs
- value_expected = true if match.index(?\n)
- tokens << [match, kind]
- next
-
- elsif match = scan(/\\?\n/)
- kind = :space
- if match == "\n"
- value_expected = true
- state = :initial if state == :undef_comma_expected
- end
- if heredocs
- unscan # heredoc scanning needs \n at start
- state = heredocs.shift
- tokens << [:open, state.type]
- heredocs = nil if heredocs.empty?
- next
- else
- match << scan(/\s*/) unless eos?
- end
- tokens << [match, kind]
- next
-
- elsif bol? && match = scan(/\#!.*/)
- tokens << [match, :doctype]
- next
-
- elsif match = scan(/\#.*/) or
- ( bol? and match = scan(/#{patterns::RUBYDOC_OR_DATA}/o) )
- kind = :comment
- tokens << [match, kind]
- next
-
- elsif state == :initial
-
- # IDENTS #
- if match = scan(unicode ? /#{patterns::METHOD_NAME}/uo :
- /#{patterns::METHOD_NAME}/o)
- if last_token_dot
- kind = if match[/^[A-Z]/] and not match?(/\(/) then :constant else :ident end
- else
- kind = patterns::IDENT_KIND[match]
- if kind == :ident and match[/^[A-Z]/] and not match[/[!?]$/] and not match?(/\(/)
- kind = :constant
- elsif kind == :reserved
- state = patterns::DEF_NEW_STATE[match]
- value_expected = :set if patterns::KEYWORDS_EXPECTING_VALUE[match]
- end
- end
- value_expected = :set if check(/#{patterns::VALUE_FOLLOWS}/o)
-
- elsif last_token_dot and match = scan(/#{patterns::METHOD_NAME_OPERATOR}|\(/o)
- kind = :ident
- value_expected = :set if check(/#{patterns::VALUE_FOLLOWS}/o)
-
- # OPERATORS #
- elsif not last_token_dot and match = scan(/ \.\.\.? | (?:\.|::)() | [,\(\)\[\]\{\}] | ==?=? /x)
- if match !~ / [.\)\]\}] /x or match =~ /\.\.\.?/
- value_expected = :set
- end
- last_token_dot = :set if self[1]
- kind = :operator
- unless inline_block_stack.empty?
- case match
- when '{'
- depth += 1
- when '}'
- depth -= 1
- if depth == 0 # closing brace of inline block reached
- state, depth, heredocs = inline_block_stack.pop
- heredocs = nil if heredocs && heredocs.empty?
- tokens << [match, :inline_delimiter]
- kind = :inline
- match = :close
- end
- end
- end
-
- elsif match = scan(/ ['"] /mx)
- tokens << [:open, :string]
- kind = :delimiter
- state = patterns::StringState.new :string, match == '"', match # important for streaming
-
- elsif match = scan(/#{patterns::INSTANCE_VARIABLE}/o)
- kind = :instance_variable
-
- elsif value_expected and match = scan(/\//)
- tokens << [:open, :regexp]
- kind = :delimiter
- interpreted = true
- state = patterns::StringState.new :regexp, interpreted, match
-
- # elsif match = scan(/[-+]?#{patterns::NUMERIC}/o)
- elsif match = value_expected ? scan(/[-+]?#{patterns::NUMERIC}/o) : scan(/#{patterns::NUMERIC}/o)
- kind = self[1] ? :float : :integer
-
- elsif match = scan(/#{patterns::SYMBOL}/o)
- case delim = match[1]
- when ?', ?"
- tokens << [:open, :symbol]
- tokens << [':', :symbol]
- match = delim.chr
- kind = :delimiter
- state = patterns::StringState.new :symbol, delim == ?", match
- else
- kind = :symbol
- end
-
- elsif match = scan(/ [-+!~^]=? | [*|&]{1,2}=? | >>? /x)
- value_expected = :set
- kind = :operator
-
- elsif value_expected and match = scan(/#{patterns::HEREDOC_OPEN}/o)
- indented = self[1] == '-'
- quote = self[3]
- delim = self[quote ? 4 : 2]
- kind = patterns::QUOTE_TO_TYPE[quote]
- tokens << [:open, kind]
- tokens << [match, :delimiter]
- match = :close
- heredoc = patterns::StringState.new kind, quote != '\'', delim, (indented ? :indented : :linestart )
- heredocs ||= [] # create heredocs if empty
- heredocs << heredoc
-
- elsif value_expected and match = scan(/#{patterns::FANCY_START_CORRECT}/o)
- kind, interpreted = *patterns::FancyStringType.fetch(self[1]) do
- raise_inspect 'Unknown fancy string: %%%p' % k, tokens
- end
- tokens << [:open, kind]
- state = patterns::StringState.new kind, interpreted, self[2]
- kind = :delimiter
-
- elsif value_expected and match = scan(/#{patterns::CHARACTER}/o)
- kind = :integer
-
- elsif match = scan(/ [\/%]=? | <(?:<|=>?)? | [?:;] /x)
- value_expected = :set
- kind = :operator
-
- elsif match = scan(/`/)
- if last_token_dot
- kind = :operator
- else
- tokens << [:open, :shell]
- kind = :delimiter
- state = patterns::StringState.new :shell, true, match
- end
-
- elsif match = scan(/#{patterns::GLOBAL_VARIABLE}/o)
- kind = :global_variable
-
- elsif match = scan(/#{patterns::CLASS_VARIABLE}/o)
- kind = :class_variable
-
- else
- if !unicode
- # check for unicode
- debug, $DEBUG = $DEBUG, false
- begin
- if check(/./mu).size > 1
- # seems like we should try again with unicode
- unicode = true
- end
- rescue
- # bad unicode char; use getch
- ensure
- $DEBUG = debug
- end
- next if unicode
- end
- kind = :error
- match = getch
-
- end
-
- elsif state == :def_expected
- state = :initial
- if scan(/self\./)
- tokens << ['self', :pre_constant]
- tokens << ['.', :operator]
- end
- if match = scan(unicode ? /(?>#{patterns::METHOD_NAME_EX})(?!\.|::)/uo :
- /(?>#{patterns::METHOD_NAME_EX})(?!\.|::)/o)
- kind = :method
- else
- next
- end
-
- elsif state == :module_expected
- if match = scan(/<</)
- kind = :operator
- else
- state = :initial
- if match = scan(/ (?:#{patterns::IDENT}::)* #{patterns::IDENT} /ox)
- kind = :class
- else
- next
- end
- end
-
- elsif state == :undef_expected
- state = :undef_comma_expected
- if match = scan(/#{patterns::METHOD_NAME_EX}/o)
- kind = :method
- elsif match = scan(/#{patterns::SYMBOL}/o)
- case delim = match[1]
- when ?', ?"
- tokens << [:open, :symbol]
- tokens << [':', :symbol]
- match = delim.chr
- kind = :delimiter
- state = patterns::StringState.new :symbol, delim == ?", match
- state.next_state = :undef_comma_expected
- else
- kind = :symbol
- end
- else
- state = :initial
- next
- end
-
- elsif state == :alias_expected
- match = scan(unicode ? /(#{patterns::METHOD_NAME_OR_SYMBOL})([ \t]+)(#{patterns::METHOD_NAME_OR_SYMBOL})/uo :
- /(#{patterns::METHOD_NAME_OR_SYMBOL})([ \t]+)(#{patterns::METHOD_NAME_OR_SYMBOL})/o)
-
- if match
- tokens << [self[1], (self[1][0] == ?: ? :symbol : :method)]
- tokens << [self[2], :space]
- tokens << [self[3], (self[3][0] == ?: ? :symbol : :method)]
- end
- state = :initial
- next
-
- elsif state == :undef_comma_expected
- if match = scan(/,/)
- kind = :operator
- state = :undef_expected
- else
- state = :initial
- next
- end
-
- end
-# }}}
-
- unless kind == :error
- value_expected = value_expected == :set
- last_token_dot = last_token_dot == :set
- end
-
- if $CODERAY_DEBUG and not kind
- raise_inspect 'Error token %p in line %d' %
- [[match, kind], line], tokens, state
- end
- raise_inspect 'Empty token', tokens unless match
-
- tokens << [match, kind]
-
- if last_state
- state = last_state
- last_state = nil
- end
- end
- end
-
- inline_block_stack << [state] if state.is_a? patterns::StringState
- until inline_block_stack.empty?
- this_block = inline_block_stack.pop
- tokens << [:close, :inline] if this_block.size > 1
- state = this_block.first
- tokens << [:close, state.type]
- end
-
- tokens
- end
-
- end
-
-end
-end
-
-# vim:fdm=marker
+++ /dev/null
-# encoding: utf-8
-module CodeRay
-module Scanners
-
- module Ruby::Patterns # :nodoc:
-
- RESERVED_WORDS = %w[
- and def end in or unless begin
- defined? ensure module redo super until
- BEGIN break do next rescue then
- when END case else for retry
- while alias class elsif if not return
- undef yield
- ]
-
- DEF_KEYWORDS = %w[ def ]
- UNDEF_KEYWORDS = %w[ undef ]
- ALIAS_KEYWORDS = %w[ alias ]
- MODULE_KEYWORDS = %w[class module]
- DEF_NEW_STATE = WordList.new(:initial).
- add(DEF_KEYWORDS, :def_expected).
- add(UNDEF_KEYWORDS, :undef_expected).
- add(ALIAS_KEYWORDS, :alias_expected).
- add(MODULE_KEYWORDS, :module_expected)
-
- PREDEFINED_CONSTANTS = %w[
- nil true false self
- DATA ARGV ARGF __FILE__ __LINE__
- ]
-
- IDENT_KIND = WordList.new(:ident).
- add(RESERVED_WORDS, :reserved).
- add(PREDEFINED_CONSTANTS, :pre_constant)
-
- IDENT = 'ä'[/[[:alpha:]]/] == 'ä' ? /[[:alpha:]_][[:alnum:]_]*/ : /[^\W\d]\w*/
-
- METHOD_NAME = / #{IDENT} [?!]? /ox
- METHOD_NAME_OPERATOR = /
- \*\*? # multiplication and power
- | [-+~]@? # plus, minus, tilde with and without at sign
- | [\/%&|^`] # division, modulo or format strings, and, or, xor, system
- | \[\]=? # array getter and setter
- | << | >> # append or shift left, shift right
- | <=?>? | >=? # comparison, rocket operator
- | ===? | =~ # simple equality, case equality, match
- | ![~=@]? # negation with and without at sign, not-equal and not-match
- /ox
- METHOD_NAME_EX = / #{IDENT} (?:[?!]|=(?!>))? | #{METHOD_NAME_OPERATOR} /ox
- INSTANCE_VARIABLE = / @ #{IDENT} /ox
- CLASS_VARIABLE = / @@ #{IDENT} /ox
- OBJECT_VARIABLE = / @@? #{IDENT} /ox
- GLOBAL_VARIABLE = / \$ (?: #{IDENT} | [1-9]\d* | 0\w* | [~&+`'=\/,;_.<>!@$?*":\\] | -[a-zA-Z_0-9] ) /ox
- PREFIX_VARIABLE = / #{GLOBAL_VARIABLE} | #{OBJECT_VARIABLE} /ox
- VARIABLE = / @?@? #{IDENT} | #{GLOBAL_VARIABLE} /ox
-
- QUOTE_TO_TYPE = {
- '`' => :shell,
- '/'=> :regexp,
- }
- QUOTE_TO_TYPE.default = :string
-
- REGEXP_MODIFIERS = /[mixounse]*/
- REGEXP_SYMBOLS = /[|?*+(){}\[\].^$]/
-
- DECIMAL = /\d+(?:_\d+)*/
- OCTAL = /0_?[0-7]+(?:_[0-7]+)*/
- HEXADECIMAL = /0x[0-9A-Fa-f]+(?:_[0-9A-Fa-f]+)*/
- BINARY = /0b[01]+(?:_[01]+)*/
-
- EXPONENT = / [eE] [+-]? #{DECIMAL} /ox
- FLOAT_SUFFIX = / #{EXPONENT} | \. #{DECIMAL} #{EXPONENT}? /ox
- FLOAT_OR_INT = / #{DECIMAL} (?: #{FLOAT_SUFFIX} () )? /ox
- NUMERIC = / (?: (?=0) (?: #{OCTAL} | #{HEXADECIMAL} | #{BINARY} ) | #{FLOAT_OR_INT} ) /ox
-
- SYMBOL = /
- :
- (?:
- #{METHOD_NAME_EX}
- | #{PREFIX_VARIABLE}
- | ['"]
- )
- /ox
- METHOD_NAME_OR_SYMBOL = / #{METHOD_NAME_EX} | #{SYMBOL} /ox
-
- SIMPLE_ESCAPE = /
- [abefnrstv]
- | [0-7]{1,3}
- | x[0-9A-Fa-f]{1,2}
- | .?
- /mx
-
- CONTROL_META_ESCAPE = /
- (?: M-|C-|c )
- (?: \\ (?: M-|C-|c ) )*
- (?: [^\\] | \\ #{SIMPLE_ESCAPE} )?
- /mox
-
- ESCAPE = /
- #{CONTROL_META_ESCAPE} | #{SIMPLE_ESCAPE}
- /mox
-
- CHARACTER = /
- \?
- (?:
- [^\s\\]
- | \\ #{ESCAPE}
- )
- /mox
-
- # NOTE: This is not completely correct, but
- # nobody needs heredoc delimiters ending with \n.
- HEREDOC_OPEN = /
- << (-)? # $1 = float
- (?:
- ( [A-Za-z_0-9]+ ) # $2 = delim
- |
- ( ["'`\/] ) # $3 = quote, type
- ( [^\n]*? ) \3 # $4 = delim
- )
- /mx
-
- RUBYDOC = /
- =begin (?!\S)
- .*?
- (?: \Z | ^=end (?!\S) [^\n]* )
- /mx
-
- DATA = /
- __END__$
- .*?
- (?: \Z | (?=^\#CODE) )
- /mx
-
- # Checks for a valid value to follow. This enables
- # value_expected in method calls without parentheses.
- VALUE_FOLLOWS = /
- (?>[ \t\f\v]+)
- (?:
- [%\/][^\s=]
- | <<-?\S
- | [-+] \d
- | #{CHARACTER}
- )
- /x
- KEYWORDS_EXPECTING_VALUE = WordList.new.add(%w[
- and end in or unless begin
- defined? ensure redo super until
- break do next rescue then
- when case else for retry
- while elsif if not return
- yield
- ])
-
- RUBYDOC_OR_DATA = / #{RUBYDOC} | #{DATA} /xo
-
- RDOC_DATA_START = / ^=begin (?!\S) | ^__END__$ /x
-
- FANCY_START_CORRECT = / % ( [qQwWxsr] | (?![a-zA-Z0-9]) ) ([^a-zA-Z0-9]) /mx
-
- FancyStringType = {
- 'q' => [:string, false],
- 'Q' => [:string, true],
- 'r' => [:regexp, true],
- 's' => [:symbol, false],
- 'x' => [:shell, true]
- }
- FancyStringType['w'] = FancyStringType['q']
- FancyStringType['W'] = FancyStringType[''] = FancyStringType['Q']
-
- class StringState < Struct.new :type, :interpreted, :delim, :heredoc,
- :paren, :paren_depth, :pattern, :next_state
-
- CLOSING_PAREN = Hash[ *%w[
- ( )
- [ ]
- < >
- { }
- ] ]
-
- CLOSING_PAREN.each { |k,v| k.freeze; v.freeze } # debug, if I try to change it with <<
- OPENING_PAREN = CLOSING_PAREN.invert
-
- STRING_PATTERN = Hash.new do |h, k|
- delim, interpreted = *k
- delim_pattern = Regexp.escape(delim.dup) # dup: workaround for old Ruby
- if closing_paren = CLOSING_PAREN[delim]
- delim_pattern = delim_pattern[0..-1] if defined? JRUBY_VERSION # JRuby fix
- delim_pattern << Regexp.escape(closing_paren)
- end
- delim_pattern << '\\\\' unless delim == '\\'
-
- special_escapes =
- case interpreted
- when :regexp_symbols
- '| ' + REGEXP_SYMBOLS.source
- when :words
- '| \s'
- end
-
- h[k] =
- if interpreted and not delim == '#'
- / (?= [#{delim_pattern}] | \# [{$@] #{special_escapes} ) /mx
- else
- / (?= [#{delim_pattern}] #{special_escapes} ) /mx
- end
- end
-
- HEREDOC_PATTERN = Hash.new do |h, k|
- delim, interpreted, indented = *k
- delim_pattern = Regexp.escape(delim.dup) # dup: workaround for old Ruby
- delim_pattern = / \n #{ '(?>[\ \t]*)' if indented } #{ Regexp.new delim_pattern } $ /x
- h[k] =
- if interpreted
- / (?= #{delim_pattern}() | \\ | \# [{$@] ) /mx # $1 set == end of heredoc
- else
- / (?= #{delim_pattern}() | \\ ) /mx
- end
- end
-
- def initialize kind, interpreted, delim, heredoc = false
- if heredoc
- pattern = HEREDOC_PATTERN[ [delim, interpreted, heredoc == :indented] ]
- delim = nil
- else
- pattern = STRING_PATTERN[ [delim, interpreted] ]
- if paren = CLOSING_PAREN[delim]
- delim, paren = paren, delim
- paren_depth = 1
- end
- end
- super kind, interpreted, delim, heredoc, paren, paren_depth, pattern, :initial
- end
- end unless defined? StringState
-
- end
-
-end
-end
+++ /dev/null
-module CodeRay
- module Scanners
-
- # Scheme scanner for CodeRay (by closure).
- # Thanks to murphy for putting CodeRay into public.
- class Scheme < Scanner
-
- # TODO: function defs
- # TODO: built-in functions
-
- register_for :scheme
- file_extension 'scm'
-
- CORE_FORMS = %w[
- lambda let let* letrec syntax-case define-syntax let-syntax
- letrec-syntax begin define quote if or and cond case do delay
- quasiquote set! cons force call-with-current-continuation call/cc
- ]
-
- IDENT_KIND = CaseIgnoringWordList.new(:ident).
- add(CORE_FORMS, :reserved)
-
- #IDENTIFIER_INITIAL = /[a-z!@\$%&\*\/\:<=>\?~_\^]/i
- #IDENTIFIER_SUBSEQUENT = /#{IDENTIFIER_INITIAL}|\d|\.|\+|-/
- #IDENTIFIER = /#{IDENTIFIER_INITIAL}#{IDENTIFIER_SUBSEQUENT}*|\+|-|\.{3}/
- IDENTIFIER = /[a-zA-Z!@$%&*\/:<=>?~_^][\w!@$%&*\/:<=>?~^.+\-]*|[+-]|\.\.\./
- DIGIT = /\d/
- DIGIT10 = DIGIT
- DIGIT16 = /[0-9a-f]/i
- DIGIT8 = /[0-7]/
- DIGIT2 = /[01]/
- RADIX16 = /\#x/i
- RADIX8 = /\#o/i
- RADIX2 = /\#b/i
- RADIX10 = /\#d/i
- EXACTNESS = /#i|#e/i
- SIGN = /[\+-]?/
- EXP_MARK = /[esfdl]/i
- EXP = /#{EXP_MARK}#{SIGN}#{DIGIT}+/
- SUFFIX = /#{EXP}?/
- PREFIX10 = /#{RADIX10}?#{EXACTNESS}?|#{EXACTNESS}?#{RADIX10}?/
- PREFIX16 = /#{RADIX16}#{EXACTNESS}?|#{EXACTNESS}?#{RADIX16}/
- PREFIX8 = /#{RADIX8}#{EXACTNESS}?|#{EXACTNESS}?#{RADIX8}/
- PREFIX2 = /#{RADIX2}#{EXACTNESS}?|#{EXACTNESS}?#{RADIX2}/
- UINT10 = /#{DIGIT10}+#*/
- UINT16 = /#{DIGIT16}+#*/
- UINT8 = /#{DIGIT8}+#*/
- UINT2 = /#{DIGIT2}+#*/
- DECIMAL = /#{DIGIT10}+#+\.#*#{SUFFIX}|#{DIGIT10}+\.#{DIGIT10}*#*#{SUFFIX}|\.#{DIGIT10}+#*#{SUFFIX}|#{UINT10}#{EXP}/
- UREAL10 = /#{UINT10}\/#{UINT10}|#{DECIMAL}|#{UINT10}/
- UREAL16 = /#{UINT16}\/#{UINT16}|#{UINT16}/
- UREAL8 = /#{UINT8}\/#{UINT8}|#{UINT8}/
- UREAL2 = /#{UINT2}\/#{UINT2}|#{UINT2}/
- REAL10 = /#{SIGN}#{UREAL10}/
- REAL16 = /#{SIGN}#{UREAL16}/
- REAL8 = /#{SIGN}#{UREAL8}/
- REAL2 = /#{SIGN}#{UREAL2}/
- IMAG10 = /i|#{UREAL10}i/
- IMAG16 = /i|#{UREAL16}i/
- IMAG8 = /i|#{UREAL8}i/
- IMAG2 = /i|#{UREAL2}i/
- COMPLEX10 = /#{REAL10}@#{REAL10}|#{REAL10}\+#{IMAG10}|#{REAL10}-#{IMAG10}|\+#{IMAG10}|-#{IMAG10}|#{REAL10}/
- COMPLEX16 = /#{REAL16}@#{REAL16}|#{REAL16}\+#{IMAG16}|#{REAL16}-#{IMAG16}|\+#{IMAG16}|-#{IMAG16}|#{REAL16}/
- COMPLEX8 = /#{REAL8}@#{REAL8}|#{REAL8}\+#{IMAG8}|#{REAL8}-#{IMAG8}|\+#{IMAG8}|-#{IMAG8}|#{REAL8}/
- COMPLEX2 = /#{REAL2}@#{REAL2}|#{REAL2}\+#{IMAG2}|#{REAL2}-#{IMAG2}|\+#{IMAG2}|-#{IMAG2}|#{REAL2}/
- NUM10 = /#{PREFIX10}?#{COMPLEX10}/
- NUM16 = /#{PREFIX16}#{COMPLEX16}/
- NUM8 = /#{PREFIX8}#{COMPLEX8}/
- NUM2 = /#{PREFIX2}#{COMPLEX2}/
- NUM = /#{NUM10}|#{NUM16}|#{NUM8}|#{NUM2}/
-
- private
- def scan_tokens tokens,options
-
- state = :initial
- ident_kind = IDENT_KIND
-
- until eos?
- kind = match = nil
-
- case state
- when :initial
- if scan(/ \s+ | \\\n /x)
- kind = :space
- elsif scan(/['\(\[\)\]]|#\(/)
- kind = :operator_fat
- elsif scan(/;.*/)
- kind = :comment
- elsif scan(/#\\(?:newline|space|.?)/)
- kind = :char
- elsif scan(/#[ft]/)
- kind = :pre_constant
- elsif scan(/#{IDENTIFIER}/o)
- kind = ident_kind[matched]
- elsif scan(/\./)
- kind = :operator
- elsif scan(/"/)
- tokens << [:open, :string]
- state = :string
- tokens << ['"', :delimiter]
- next
- elsif scan(/#{NUM}/o) and not matched.empty?
- kind = :integer
- elsif getch
- kind = :error
- end
-
- when :string
- if scan(/[^"\\]+/) or scan(/\\.?/)
- kind = :content
- elsif scan(/"/)
- tokens << ['"', :delimiter]
- tokens << [:close, :string]
- state = :initial
- next
- else
- raise_inspect "else case \" reached; %p not handled." % peek(1),
- tokens, state
- end
-
- else
- raise "else case reached"
- end
-
- match ||= matched
- if $CODERAY_DEBUG and not kind
- raise_inspect 'Error token %p in line %d' %
- [[match, kind], line], tokens
- end
- raise_inspect 'Empty token', tokens, state unless match
-
- tokens << [match, kind]
-
- end # until eos
-
- if state == :string
- tokens << [:close, :string]
- end
-
- tokens
-
- end #scan_tokens
- end #class
- end #module scanners
-end #module coderay
\ No newline at end of file
+++ /dev/null
-module CodeRay module Scanners
-
- # by Josh Goebel
- class SQL < Scanner
-
- register_for :sql
-
- RESERVED_WORDS = %w(
- create database table index trigger drop primary key set select
- insert update delete replace into
- on from values before and or if exists case when
- then else as group order by avg where
- join inner outer union engine not
- like end using collate show columns begin
- )
-
- PREDEFINED_TYPES = %w(
- char varchar enum binary text tinytext mediumtext
- longtext blob tinyblob mediumblob longblob timestamp
- date time datetime year double decimal float int
- integer tinyint mediumint bigint smallint unsigned bit
- bool boolean hex bin oct
- )
-
- PREDEFINED_FUNCTIONS = %w( sum cast abs pi count min max avg )
-
- DIRECTIVES = %w( auto_increment unique default charset )
-
- PREDEFINED_CONSTANTS = %w( null true false )
-
- IDENT_KIND = CaseIgnoringWordList.new(:ident).
- add(RESERVED_WORDS, :reserved).
- add(PREDEFINED_TYPES, :pre_type).
- add(PREDEFINED_CONSTANTS, :pre_constant).
- add(PREDEFINED_FUNCTIONS, :predefined).
- add(DIRECTIVES, :directive)
-
- ESCAPE = / [rbfntv\n\\\/'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} | . /mx
- UNICODE_ESCAPE = / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} /x
-
- STRING_PREFIXES = /[xnb]|_\w+/i
-
- def scan_tokens tokens, options
-
- state = :initial
- string_type = nil
- string_content = ''
-
- until eos?
-
- kind = nil
- match = nil
-
- if state == :initial
-
- if scan(/ \s+ | \\\n /x)
- kind = :space
-
- elsif scan(/^(?:--\s?|#).*/)
- kind = :comment
-
- elsif scan(%r! /\* (?: .*? \*/ | .* ) !mx)
- kind = :comment
-
- elsif scan(/ [-+*\/=<>;,!&^|()\[\]{}~%] | \.(?!\d) /x)
- kind = :operator
-
- elsif scan(/(#{STRING_PREFIXES})?([`"'])/o)
- prefix = self[1]
- string_type = self[2]
- tokens << [:open, :string]
- tokens << [prefix, :modifier] if prefix
- match = string_type
- state = :string
- kind = :delimiter
-
- elsif match = scan(/ @? [A-Za-z_][A-Za-z_0-9]* /x)
- kind = match[0] == ?@ ? :variable : IDENT_KIND[match.downcase]
-
- elsif scan(/0[xX][0-9A-Fa-f]+/)
- kind = :hex
-
- elsif scan(/0[0-7]+(?![89.eEfF])/)
- kind = :oct
-
- elsif scan(/(?>\d+)(?![.eEfF])/)
- kind = :integer
-
- elsif scan(/\d[fF]|\d*\.\d+(?:[eE][+-]?\d+)?|\d+[eE][+-]?\d+/)
- kind = :float
-
- else
- getch
- kind = :error
-
- end
-
- elsif state == :string
- if match = scan(/[^\\"'`]+/)
- string_content << match
- next
- elsif match = scan(/["'`]/)
- if string_type == match
- if peek(1) == string_type # doubling means escape
- string_content << string_type << getch
- next
- end
- unless string_content.empty?
- tokens << [string_content, :content]
- string_content = ''
- end
- tokens << [matched, :delimiter]
- tokens << [:close, :string]
- state = :initial
- string_type = nil
- next
- else
- string_content << match
- end
- next
- elsif scan(/ \\ (?: #{ESCAPE} | #{UNICODE_ESCAPE} ) /mox)
- unless string_content.empty?
- tokens << [string_content, :content]
- string_content = ''
- end
- kind = :char
- elsif match = scan(/ \\ . /mox)
- string_content << match
- next
- elsif scan(/ \\ | $ /x)
- unless string_content.empty?
- tokens << [string_content, :content]
- string_content = ''
- end
- kind = :error
- state = :initial
- else
- raise "else case \" reached; %p not handled." % peek(1), tokens
- end
-
- else
- raise 'else-case reached', tokens
-
- end
-
- match ||= matched
- unless kind
- raise_inspect 'Error token %p in line %d' %
- [[match, kind], line], tokens, state
- end
- raise_inspect 'Empty token', tokens unless match
-
- tokens << [match, kind]
-
- end
- tokens
-
- end
-
- end
-
-end end
\ No newline at end of file
+++ /dev/null
-module CodeRay
-module Scanners
-
- load :html
-
- # XML Scanner
- #
- # Currently this is the same scanner as Scanners::HTML.
- class XML < HTML
-
- register_for :xml
- file_extension 'xml'
-
- end
-
-end
-end
+++ /dev/null
-module CodeRay
-module Scanners
-
- # YAML Scanner
- #
- # Based on the YAML scanner from Syntax by Jamis Buck.
- class YAML < Scanner
-
- register_for :yaml
- file_extension 'yml'
-
- KINDS_NOT_LOC = :all
-
- def scan_tokens tokens, options
-
- value_expected = nil
- state = :initial
- key_indent = indent = 0
-
- until eos?
-
- kind = nil
- match = nil
- key_indent = nil if bol?
-
- if match = scan(/ +[\t ]*/)
- kind = :space
-
- elsif match = scan(/\n+/)
- kind = :space
- state = :initial if match.index(?\n)
-
- elsif match = scan(/#.*/)
- kind = :comment
-
- elsif bol? and case
- when match = scan(/---|\.\.\./)
- tokens << [:open, :head]
- tokens << [match, :head]
- tokens << [:close, :head]
- next
- when match = scan(/%.*/)
- tokens << [match, :doctype]
- next
- end
-
- elsif state == :value and case
- when !check(/(?:"[^"]*")(?=: |:$)/) && scan(/"/)
- tokens << [:open, :string]
- tokens << [matched, :delimiter]
- tokens << [matched, :content] if scan(/ [^"\\]* (?: \\. [^"\\]* )* /mx)
- tokens << [matched, :delimiter] if scan(/"/)
- tokens << [:close, :string]
- next
- when match = scan(/[|>][-+]?/)
- tokens << [:open, :string]
- tokens << [match, :delimiter]
- string_indent = key_indent || column(pos - match.size - 1)
- tokens << [matched, :content] if scan(/(?:\n+ {#{string_indent + 1}}.*)+/)
- tokens << [:close, :string]
- next
- when match = scan(/(?![!"*&]).+?(?=$|\s+#)/)
- tokens << [match, :string]
- string_indent = key_indent || column(pos - match.size - 1)
- tokens << [matched, :string] if scan(/(?:\n+ {#{string_indent + 1}}.*)+/)
- next
- end
-
- elsif case
- when match = scan(/[-:](?= |$)/)
- state = :value if state == :colon && (match == ':' || match == '-')
- state = :value if state == :initial && match == '-'
- kind = :operator
- when match = scan(/[,{}\[\]]/)
- kind = :operator
- when state == :initial && match = scan(/[\w.() ]*\S(?=: |:$)/)
- kind = :key
- key_indent = column(pos - match.size - 1)
- # tokens << [key_indent.inspect, :debug]
- state = :colon
- when match = scan(/(?:"[^"\n]*"|'[^'\n]*')(?=: |:$)/)
- tokens << [:open, :key]
- tokens << [match[0,1], :delimiter]
- tokens << [match[1..-2], :content]
- tokens << [match[-1,1], :delimiter]
- tokens << [:close, :key]
- key_indent = column(pos - match.size - 1)
- # tokens << [key_indent.inspect, :debug]
- state = :colon
- next
- when scan(/(![\w\/]+)(:([\w:]+))?/)
- tokens << [self[1], :type]
- if self[2]
- tokens << [':', :operator]
- tokens << [self[3], :class]
- end
- next
- when scan(/&\S+/)
- kind = :variable
- when scan(/\*\w+/)
- kind = :global_variable
- when scan(/<</)
- kind = :class_variable
- when scan(/\d\d:\d\d:\d\d/)
- kind = :oct
- when scan(/\d\d\d\d-\d\d-\d\d\s\d\d:\d\d:\d\d(\.\d+)? [-+]\d\d:\d\d/)
- kind = :oct
- when scan(/:\w+/)
- kind = :symbol
- when scan(/[^:\s]+(:(?! |$)[^:\s]*)* .*/)
- kind = :error
- when scan(/[^:\s]+(:(?! |$)[^:\s]*)*/)
- kind = :error
- end
-
- else
- getch
- kind = :error
-
- end
-
- match ||= matched
-
- if $CODERAY_DEBUG and not kind
- raise_inspect 'Error token %p in line %d' %
- [[match, kind], line], tokens, state
- end
- raise_inspect 'Empty token', tokens, state unless match
-
- tokens << [match, kind]
-
- end
-
- tokens
- end
-
- end
-
-end
-end
+++ /dev/null
-module CodeRay
-
- # This module holds the Style class and its subclasses.
- #
- # See Plugin.
- module Styles
- extend PluginHost
- plugin_path File.dirname(__FILE__), 'styles'
-
- class Style
- extend Plugin
- plugin_host Styles
-
- DEFAULT_OPTIONS = { }
-
- end
-
- end
-
-end
+++ /dev/null
-module CodeRay
-module Styles
-
- default :cycnus
-
-end
-end
+++ /dev/null
-module CodeRay
-module Styles
-
- class Cycnus < Style
-
- register_for :cycnus
-
- code_background = '#f8f8f8'
- numbers_background = '#def'
- border_color = 'silver'
- normal_color = '#000'
-
- CSS_MAIN_STYLES = <<-MAIN
-.CodeRay {
- background-color: #{code_background};
- border: 1px solid #{border_color};
- font-family: 'Courier New', 'Terminal', monospace;
- color: #{normal_color};
-}
-.CodeRay pre { margin: 0px }
-
-div.CodeRay { }
-
-span.CodeRay { white-space: pre; border: 0px; padding: 2px }
-
-table.CodeRay { border-collapse: collapse; width: 100%; padding: 2px }
-table.CodeRay td { padding: 2px 4px; vertical-align: top }
-
-.CodeRay .line_numbers, .CodeRay .no {
- background-color: #{numbers_background};
- color: gray;
- text-align: right;
-}
-.CodeRay .line_numbers tt { font-weight: bold }
-.CodeRay .line_numbers .highlighted { color: red }
-.CodeRay .no { padding: 0px 4px }
-.CodeRay .code { width: 100% }
-
-ol.CodeRay { font-size: 10pt }
-ol.CodeRay li { white-space: pre }
-
-.CodeRay .code pre { overflow: auto }
- MAIN
-
- TOKEN_COLORS = <<-'TOKENS'
-.debug { color:white ! important; background:blue ! important; }
-
-.af { color:#00C }
-.an { color:#007 }
-.at { color:#f08 }
-.av { color:#700 }
-.aw { color:#C00 }
-.bi { color:#509; font-weight:bold }
-.c { color:#888; }
-
-.ch { color:#04D }
-.ch .k { color:#04D }
-.ch .dl { color:#039 }
-
-.cl { color:#B06; font-weight:bold }
-.cm { color:#A08; font-weight:bold }
-.co { color:#036; font-weight:bold }
-.cr { color:#0A0 }
-.cv { color:#369 }
-.de { color:#B0B; }
-.df { color:#099; font-weight:bold }
-.di { color:#088; font-weight:bold }
-.dl { color:black }
-.do { color:#970 }
-.dt { color:#34b }
-.ds { color:#D42; font-weight:bold }
-.e { color:#666; font-weight:bold }
-.en { color:#800; font-weight:bold }
-.er { color:#F00; background-color:#FAA }
-.ex { color:#C00; font-weight:bold }
-.fl { color:#60E; font-weight:bold }
-.fu { color:#06B; font-weight:bold }
-.gv { color:#d70; font-weight:bold }
-.hx { color:#058; font-weight:bold }
-.i { color:#00D; font-weight:bold }
-.ic { color:#B44; font-weight:bold }
-
-.il { background: #ddd; color: black }
-.il .il { background: #ccc }
-.il .il .il { background: #bbb }
-.il .idl { background: #ddd; font-weight: bold; color: #666 }
-.idl { background-color: #bbb; font-weight: bold; color: #666; }
-
-.im { color:#f00; }
-.in { color:#B2B; font-weight:bold }
-.iv { color:#33B }
-.la { color:#970; font-weight:bold }
-.lv { color:#963 }
-.oc { color:#40E; font-weight:bold }
-.of { color:#000; font-weight:bold }
-.op { }
-.pc { color:#038; font-weight:bold }
-.pd { color:#369; font-weight:bold }
-.pp { color:#579; }
-.ps { color:#00C; font-weight:bold }
-.pt { color:#074; font-weight:bold }
-.r, .kw { color:#080; font-weight:bold }
-
-.ke { color: #808; }
-.ke .dl { color: #606; }
-.ke .ch { color: #80f; }
-.vl { color: #088; }
-
-.rx { background-color:#fff0ff }
-.rx .k { color:#808 }
-.rx .dl { color:#404 }
-.rx .mod { color:#C2C }
-.rx .fu { color:#404; font-weight: bold }
-
-.s { background-color:#fff0f0; color: #D20; }
-.s .s { background-color:#ffe0e0 }
-.s .s .s { background-color:#ffd0d0 }
-.s .k { }
-.s .ch { color: #b0b; }
-.s .dl { color: #710; }
-
-.sh { background-color:#f0fff0; color:#2B2 }
-.sh .k { }
-.sh .dl { color:#161 }
-
-.sy { color:#A60 }
-.sy .k { color:#A60 }
-.sy .dl { color:#630 }
-
-.ta { color:#070 }
-.tf { color:#070; font-weight:bold }
-.ts { color:#D70; font-weight:bold }
-.ty { color:#339; font-weight:bold }
-.v { color:#036 }
-.xt { color:#444 }
-
-.ins { background: #afa; }
-.del { background: #faa; }
-.chg { color: #aaf; background: #007; }
-.head { color: #f8f; background: #505 }
-
-.ins .ins { color: #080; font-weight:bold }
-.del .del { color: #800; font-weight:bold }
-.chg .chg { color: #66f; }
-.head .head { color: #f4f; }
- TOKENS
-
- end
-
-end
-end
+++ /dev/null
-module CodeRay
-module Styles
-
- class Murphy < Style
-
- register_for :murphy
-
- code_background = '#001129'
- numbers_background = code_background
- border_color = 'silver'
- normal_color = '#C0C0C0'
-
- CSS_MAIN_STYLES = <<-MAIN
-.CodeRay {
- background-color: #{code_background};
- border: 1px solid #{border_color};
- font-family: 'Courier New', 'Terminal', monospace;
- color: #{normal_color};
-}
-.CodeRay pre { margin: 0px; }
-
-div.CodeRay { }
-
-span.CodeRay { white-space: pre; border: 0px; padding: 2px; }
-
-table.CodeRay { border-collapse: collapse; width: 100%; padding: 2px; }
-table.CodeRay td { padding: 2px 4px; vertical-align: top; }
-
-.CodeRay .line_numbers, .CodeRay .no {
- background-color: #{numbers_background};
- color: gray;
- text-align: right;
-}
-.CodeRay .line_numbers tt { font-weight: bold; }
-.CodeRay .no { padding: 0px 4px; }
-.CodeRay .code { width: 100%; }
-
-ol.CodeRay { font-size: 10pt; }
-ol.CodeRay li { white-space: pre; }
-
-.CodeRay .code pre { overflow: auto; }
- MAIN
-
- TOKEN_COLORS = <<-'TOKENS'
-.af { color:#00C; }
-.an { color:#007; }
-.av { color:#700; }
-.aw { color:#C00; }
-.bi { color:#509; font-weight:bold; }
-.c { color:#555; background-color: black; }
-
-.ch { color:#88F; }
-.ch .k { color:#04D; }
-.ch .dl { color:#039; }
-
-.cl { color:#e9e; font-weight:bold; }
-.co { color:#5ED; font-weight:bold; }
-.cr { color:#0A0; }
-.cv { color:#ccf; }
-.df { color:#099; font-weight:bold; }
-.di { color:#088; font-weight:bold; }
-.dl { color:black; }
-.do { color:#970; }
-.ds { color:#D42; font-weight:bold; }
-.e { color:#666; font-weight:bold; }
-.er { color:#F00; background-color:#FAA; }
-.ex { color:#F00; font-weight:bold; }
-.fl { color:#60E; font-weight:bold; }
-.fu { color:#5ed; font-weight:bold; }
-.gv { color:#f84; }
-.hx { color:#058; font-weight:bold; }
-.i { color:#66f; font-weight:bold; }
-.ic { color:#B44; font-weight:bold; }
-.il { }
-.in { color:#B2B; font-weight:bold; }
-.iv { color:#aaf; }
-.la { color:#970; font-weight:bold; }
-.lv { color:#963; }
-.oc { color:#40E; font-weight:bold; }
-.of { color:#000; font-weight:bold; }
-.op { }
-.pc { color:#08f; font-weight:bold; }
-.pd { color:#369; font-weight:bold; }
-.pp { color:#579; }
-.pt { color:#66f; font-weight:bold; }
-.r { color:#5de; font-weight:bold; }
-.r, .kw { color:#5de; font-weight:bold }
-
-.ke { color: #808; }
-
-.rx { background-color:#221133; }
-.rx .k { color:#f8f; }
-.rx .dl { color:#f0f; }
-.rx .mod { color:#f0b; }
-.rx .fu { color:#404; font-weight: bold; }
-
-.s { background-color:#331122; }
-.s .s { background-color:#ffe0e0; }
-.s .s .s { background-color:#ffd0d0; }
-.s .k { color:#F88; }
-.s .dl { color:#f55; }
-
-.sh { background-color:#f0fff0; }
-.sh .k { color:#2B2; }
-.sh .dl { color:#161; }
-
-.sy { color:#Fc8; }
-.sy .k { color:#Fc8; }
-.sy .dl { color:#F84; }
-
-.ta { color:#070; }
-.tf { color:#070; font-weight:bold; }
-.ts { color:#D70; font-weight:bold; }
-.ty { color:#339; font-weight:bold; }
-.v { color:#036; }
-.xt { color:#444; }
-
-.ins { background: #afa; }
-.del { background: #faa; }
-.chg { color: #aaf; background: #007; }
-.head { color: #f8f; background: #505 }
-
-.ins .ins { color: #080; font-weight:bold }
-.del .del { color: #800; font-weight:bold }
-.chg .chg { color: #66f; }
-.head .head { color: #f4f; }
- TOKENS
-
- end
-
-end
-end
+++ /dev/null
-module CodeRay
- class Tokens
- ClassOfKind = Hash.new do |h, k|
- h[k] = k.to_s
- end
- ClassOfKind.update with = {
- :annotation => 'at',
- :attribute_name => 'an',
- :attribute_name_fat => 'af',
- :attribute_value => 'av',
- :attribute_value_fat => 'aw',
- :bin => 'bi',
- :char => 'ch',
- :class => 'cl',
- :class_variable => 'cv',
- :color => 'cr',
- :comment => 'c',
- :complex => 'cm',
- :constant => 'co',
- :content => 'k',
- :decorator => 'de',
- :definition => 'df',
- :delimiter => 'dl',
- :directive => 'di',
- :doc => 'do',
- :doctype => 'dt',
- :doc_string => 'ds',
- :entity => 'en',
- :error => 'er',
- :escape => 'e',
- :exception => 'ex',
- :float => 'fl',
- :function => 'fu',
- :global_variable => 'gv',
- :hex => 'hx',
- :imaginary => 'cm',
- :important => 'im',
- :include => 'ic',
- :inline => 'il',
- :inline_delimiter => 'idl',
- :instance_variable => 'iv',
- :integer => 'i',
- :interpreted => 'in',
- :keyword => 'kw',
- :key => 'ke',
- :label => 'la',
- :local_variable => 'lv',
- :modifier => 'mod',
- :oct => 'oc',
- :operator_fat => 'of',
- :pre_constant => 'pc',
- :pre_type => 'pt',
- :predefined => 'pd',
- :preprocessor => 'pp',
- :pseudo_class => 'ps',
- :regexp => 'rx',
- :reserved => 'r',
- :shell => 'sh',
- :string => 's',
- :symbol => 'sy',
- :tag => 'ta',
- :tag_fat => 'tf',
- :tag_special => 'ts',
- :type => 'ty',
- :variable => 'v',
- :value => 'vl',
- :xml_text => 'xt',
-
- :insert => 'ins',
- :delete => 'del',
- :change => 'chg',
- :head => 'head',
-
- :ident => :NO_HIGHLIGHT, # 'id'
- #:operator => 'op',
- :operator => :NO_HIGHLIGHT, # 'op'
- :space => :NO_HIGHLIGHT, # 'sp'
- :plain => :NO_HIGHLIGHT,
- }
- ClassOfKind[:method] = ClassOfKind[:function]
- ClassOfKind[:open] = ClassOfKind[:close] = ClassOfKind[:delimiter]
- ClassOfKind[:nesting_delimiter] = ClassOfKind[:delimiter]
- ClassOfKind[:escape] = ClassOfKind[:delimiter]
- #ClassOfKind.default = ClassOfKind[:error] or raise 'no class found for :error!'
- end
-end
\ No newline at end of file
+++ /dev/null
-module CodeRay
-
- # = Tokens
- #
- # The Tokens class represents a list of tokens returnd from
- # a Scanner.
- #
- # A token is not a special object, just a two-element Array
- # consisting of
- # * the _token_ _kind_ (a Symbol representing the type of the token)
- # * the _token_ _text_ (the original source of the token in a String)
- #
- # A token looks like this:
- #
- # [:comment, '# It looks like this']
- # [:float, '3.1415926']
- # [:error, '$^']
- #
- # Some scanners also yield some kind of sub-tokens, represented by special
- # token texts, namely :open and :close .
- #
- # The Ruby scanner, for example, splits "a string" into:
- #
- # [
- # [:open, :string],
- # [:delimiter, '"'],
- # [:content, 'a string'],
- # [:delimiter, '"'],
- # [:close, :string]
- # ]
- #
- # Tokens is also the interface between Scanners and Encoders:
- # The input is split and saved into a Tokens object. The Encoder
- # then builds the output from this object.
- #
- # Thus, the syntax below becomes clear:
- #
- # CodeRay.scan('price = 2.59', :ruby).html
- # # the Tokens object is here -------^
- #
- # See how small it is? ;)
- #
- # Tokens gives you the power to handle pre-scanned code very easily:
- # You can convert it to a webpage, a YAML file, or dump it into a gzip'ed string
- # that you put in your DB.
- #
- # Tokens' subclass TokenStream allows streaming to save memory.
- class Tokens < Array
-
- # The Scanner instance that created the tokens.
- attr_accessor :scanner
-
- # Whether the object is a TokenStream.
- #
- # Returns false.
- def stream?
- false
- end
-
- # Iterates over all tokens.
- #
- # If a filter is given, only tokens of that kind are yielded.
- def each kind_filter = nil, &block
- unless kind_filter
- super(&block)
- else
- super() do |text, kind|
- next unless kind == kind_filter
- yield text, kind
- end
- end
- end
-
- # Iterates over all text tokens.
- # Range tokens like [:open, :string] are left out.
- #
- # Example:
- # tokens.each_text_token { |text, kind| text.replace html_escape(text) }
- def each_text_token
- each do |text, kind|
- next unless text.is_a? ::String
- yield text, kind
- end
- end
-
- # Encode the tokens using encoder.
- #
- # encoder can be
- # * a symbol like :html oder :statistic
- # * an Encoder class
- # * an Encoder object
- #
- # options are passed to the encoder.
- def encode encoder, options = {}
- unless encoder.is_a? Encoders::Encoder
- unless encoder.is_a? Class
- encoder_class = Encoders[encoder]
- end
- encoder = encoder_class.new options
- end
- encoder.encode_tokens self, options
- end
-
-
- # Turn into a string using Encoders::Text.
- #
- # +options+ are passed to the encoder if given.
- def to_s options = {}
- encode :text, options
- end
-
- # Redirects unknown methods to encoder calls.
- #
- # For example, if you call +tokens.html+, the HTML encoder
- # is used to highlight the tokens.
- def method_missing meth, options = {}
- Encoders[meth].new(options).encode_tokens self
- end
-
- # Returns the tokens compressed by joining consecutive
- # tokens of the same kind.
- #
- # This can not be undone, but should yield the same output
- # in most Encoders. It basically makes the output smaller.
- #
- # Combined with dump, it saves space for the cost of time.
- #
- # If the scanner is written carefully, this is not required -
- # for example, consecutive //-comment lines could already be
- # joined in one comment token by the Scanner.
- def optimize
- last_kind = last_text = nil
- new = self.class.new
- for text, kind in self
- if text.is_a? String
- if kind == last_kind
- last_text << text
- else
- new << [last_text, last_kind] if last_kind
- last_text = text
- last_kind = kind
- end
- else
- new << [last_text, last_kind] if last_kind
- last_kind = last_text = nil
- new << [text, kind]
- end
- end
- new << [last_text, last_kind] if last_kind
- new
- end
-
- # Compact the object itself; see optimize.
- def optimize!
- replace optimize
- end
-
- # Ensure that all :open tokens have a correspondent :close one.
- #
- # TODO: Test this!
- def fix
- tokens = self.class.new
- # Check token nesting using a stack of kinds.
- opened = []
- for type, kind in self
- case type
- when :open
- opened.push [:close, kind]
- when :begin_line
- opened.push [:end_line, kind]
- when :close, :end_line
- expected = opened.pop
- if [type, kind] != expected
- # Unexpected :close; decide what to do based on the kind:
- # - token was never opened: delete the :close (just skip it)
- next unless opened.rindex expected
- # - token was opened earlier: also close tokens in between
- tokens << token until (token = opened.pop) == expected
- end
- end
- tokens << [type, kind]
- end
- # Close remaining opened tokens
- tokens << token while token = opened.pop
- tokens
- end
-
- def fix!
- replace fix
- end
-
- # TODO: Scanner#split_into_lines
- #
- # Makes sure that:
- # - newlines are single tokens
- # (which means all other token are single-line)
- # - there are no open tokens at the end the line
- #
- # This makes it simple for encoders that work line-oriented,
- # like HTML with list-style numeration.
- def split_into_lines
- raise NotImplementedError
- end
-
- def split_into_lines!
- replace split_into_lines
- end
-
- # Dumps the object into a String that can be saved
- # in files or databases.
- #
- # The dump is created with Marshal.dump;
- # In addition, it is gzipped using GZip.gzip.
- #
- # The returned String object includes Undumping
- # so it has an #undump method. See Tokens.load.
- #
- # You can configure the level of compression,
- # but the default value 7 should be what you want
- # in most cases as it is a good compromise between
- # speed and compression rate.
- #
- # See GZip module.
- def dump gzip_level = 7
- require 'coderay/helpers/gzip_simple'
- dump = Marshal.dump self
- dump = dump.gzip gzip_level
- dump.extend Undumping
- end
-
- # The total size of the tokens.
- # Should be equal to the input size before
- # scanning.
- def text_size
- size = 0
- each_text_token do |t, k|
- size + t.size
- end
- size
- end
-
- # The total size of the tokens.
- # Should be equal to the input size before
- # scanning.
- def text
- map { |t, k| t if t.is_a? ::String }.join
- end
-
- # Include this module to give an object an #undump
- # method.
- #
- # The string returned by Tokens.dump includes Undumping.
- module Undumping
- # Calls Tokens.load with itself.
- def undump
- Tokens.load self
- end
- end
-
- # Undump the object using Marshal.load, then
- # unzip it using GZip.gunzip.
- #
- # The result is commonly a Tokens object, but
- # this is not guaranteed.
- def Tokens.load dump
- require 'coderay/helpers/gzip_simple'
- dump = dump.gunzip
- @dump = Marshal.load dump
- end
-
- end
-
-
- # = TokenStream
- #
- # The TokenStream class is a fake Array without elements.
- #
- # It redirects the method << to a block given at creation.
- #
- # This allows scanners and Encoders to use streaming (no
- # tokens are saved, the input is highlighted the same time it
- # is scanned) with the same code.
- #
- # See CodeRay.encode_stream and CodeRay.scan_stream
- class TokenStream < Tokens
-
- # Whether the object is a TokenStream.
- #
- # Returns true.
- def stream?
- true
- end
-
- # The Array is empty, but size counts the tokens given by <<.
- attr_reader :size
-
- # Creates a new TokenStream that calls +block+ whenever
- # its << method is called.
- #
- # Example:
- #
- # require 'coderay'
- #
- # token_stream = CodeRay::TokenStream.new do |kind, text|
- # puts 'kind: %s, text size: %d.' % [kind, text.size]
- # end
- #
- # token_stream << [:regexp, '/\d+/']
- # #-> kind: rexpexp, text size: 5.
- #
- def initialize &block
- raise ArgumentError, 'Block expected for streaming.' unless block
- @callback = block
- @size = 0
- end
-
- # Calls +block+ with +token+ and increments size.
- #
- # Returns self.
- def << token
- @callback.call(*token)
- @size += 1
- self
- end
-
- # This method is not implemented due to speed reasons. Use Tokens.
- def text_size
- raise NotImplementedError,
- 'This method is not implemented due to speed reasons.'
- end
-
- # A TokenStream cannot be dumped. Use Tokens.
- def dump
- raise NotImplementedError, 'A TokenStream cannot be dumped.'
- end
-
- # A TokenStream cannot be optimized. Use Tokens.
- def optimize
- raise NotImplementedError, 'A TokenStream cannot be optimized.'
- end
-
- end
-
-end
-
-if $0 == __FILE__
- $VERBOSE = true
- $: << File.join(File.dirname(__FILE__), '..')
- eval DATA.read, nil, $0, __LINE__ + 4
-end
-
-__END__
-require 'test/unit'
-
-class TokensTest < Test::Unit::TestCase
-
- def test_creation
- assert CodeRay::Tokens < Array
- tokens = nil
- assert_nothing_raised do
- tokens = CodeRay::Tokens.new
- end
- assert_kind_of Array, tokens
- end
-
- def test_adding_tokens
- tokens = CodeRay::Tokens.new
- assert_nothing_raised do
- tokens << ['string', :type]
- tokens << ['()', :operator]
- end
- assert_equal tokens.size, 2
- end
-
- def test_dump_undump
- tokens = CodeRay::Tokens.new
- assert_nothing_raised do
- tokens << ['string', :type]
- tokens << ['()', :operator]
- end
- tokens2 = nil
- assert_nothing_raised do
- tokens2 = tokens.dump.undump
- end
- assert_equal tokens, tokens2
- end
-
-end
\ No newline at end of file
projects / redmine.git / commitdiff