diff options
Diffstat (limited to 'vendor/gems/coderay-1.0.0/lib/coderay/tokens.rb')
| -rw-r--r-- | vendor/gems/coderay-1.0.0/lib/coderay/tokens.rb | 215 |
1 files changed, 215 insertions, 0 deletions
diff --git a/vendor/gems/coderay-1.0.0/lib/coderay/tokens.rb b/vendor/gems/coderay-1.0.0/lib/coderay/tokens.rb new file mode 100644 index 000000000..045cf4a0e --- /dev/null +++ b/vendor/gems/coderay-1.0.0/lib/coderay/tokens.rb @@ -0,0 +1,215 @@ +module CodeRay + + # GZip library for writing and reading token dumps. + autoload :GZip, 'coderay/helpers/gzip' + + # = Tokens TODO: Rewrite! + # + # The Tokens class represents a list of tokens returnd from + # a Scanner. + # + # A token is not a special object, just a two-element Array + # consisting of + # * the _token_ _text_ (the original source of the token in a String) or + # a _token_ _action_ (begin_group, end_group, begin_line, end_line) + # * the _token_ _kind_ (a Symbol representing the type of the token) + # + # A token looks like this: + # + # ['# It looks like this', :comment] + # ['3.1415926', :float] + # ['$^', :error] + # + # Some scanners also yield sub-tokens, represented by special + # token actions, namely begin_group and end_group. + # + # The Ruby scanner, for example, splits "a string" into: + # + # [ + # [:begin_group, :string], + # ['"', :delimiter], + # ['a string', :content], + # ['"', :delimiter], + # [:end_group, :string] + # ] + # + # Tokens is the interface between Scanners and Encoders: + # The input is split and saved into a Tokens object. The Encoder + # then builds the output from this object. + # + # Thus, the syntax below becomes clear: + # + # CodeRay.scan('price = 2.59', :ruby).html + # # the Tokens object is here -------^ + # + # See how small it is? ;) + # + # Tokens gives you the power to handle pre-scanned code very easily: + # You can convert it to a webpage, a YAML file, or dump it into a gzip'ed string + # that you put in your DB. + # + # It also allows you to generate tokens directly (without using a scanner), + # to load them from a file, and still use any Encoder that CodeRay provides. + class Tokens < Array + + # The Scanner instance that created the tokens. + attr_accessor :scanner + + # Encode the tokens using encoder. + # + # encoder can be + # * a symbol like :html oder :statistic + # * an Encoder class + # * an Encoder object + # + # options are passed to the encoder. + def encode encoder, options = {} + encoder = Encoders[encoder].new options if encoder.respond_to? :to_sym + encoder.encode_tokens self, options + end + + # Turn tokens into a string by concatenating them. + def to_s + encode CodeRay::Encoders::Encoder.new + end + + # Redirects unknown methods to encoder calls. + # + # For example, if you call +tokens.html+, the HTML encoder + # is used to highlight the tokens. + def method_missing meth, options = {} + encode meth, options + rescue PluginHost::PluginNotFound + super + end + + # Split the tokens into parts of the given +sizes+. + # + # The result will be an Array of Tokens objects. The parts have + # the text size specified by the parameter. In addition, each + # part closes all opened tokens. This is useful to insert tokens + # betweem them. + # + # This method is used by @Scanner#tokenize@ when called with an Array + # of source strings. The Diff encoder uses it for inline highlighting. + def split_into_parts *sizes + parts = [] + opened = [] + content = nil + part = Tokens.new + part_size = 0 + size = sizes.first + i = 0 + for item in self + case content + when nil + content = item + when String + if size && part_size + content.size > size # token must be cut + if part_size < size # some part of the token goes into this part + content = content.dup # content may no be safe to change + part << content.slice!(0, size - part_size) << item + end + # close all open groups and lines... + closing = opened.reverse.flatten.map do |content_or_kind| + case content_or_kind + when :begin_group + :end_group + when :begin_line + :end_line + else + content_or_kind + end + end + part.concat closing + begin + parts << part + part = Tokens.new + size = sizes[i += 1] + end until size.nil? || size > 0 + # ...and open them again. + part.concat opened.flatten + part_size = 0 + redo unless content.empty? + else + part << content << item + part_size += content.size + end + content = nil + when Symbol + case content + when :begin_group, :begin_line + opened << [content, item] + when :end_group, :end_line + opened.pop + else + raise ArgumentError, 'Unknown token action: %p, kind = %p' % [content, item] + end + part << content << item + content = nil + else + raise ArgumentError, 'Token input junk: %p, kind = %p' % [content, item] + end + end + parts << part + parts << Tokens.new while parts.size < sizes.size + parts + end + + # Dumps the object into a String that can be saved + # in files or databases. + # + # The dump is created with Marshal.dump; + # In addition, it is gzipped using GZip.gzip. + # + # The returned String object includes Undumping + # so it has an #undump method. See Tokens.load. + # + # You can configure the level of compression, + # but the default value 7 should be what you want + # in most cases as it is a good compromise between + # speed and compression rate. + # + # See GZip module. + def dump gzip_level = 7 + dump = Marshal.dump self + dump = GZip.gzip dump, gzip_level + dump.extend Undumping + end + + # Return the actual number of tokens. + def count + size / 2 + end + + # Include this module to give an object an #undump + # method. + # + # The string returned by Tokens.dump includes Undumping. + module Undumping + # Calls Tokens.load with itself. + def undump + Tokens.load self + end + end + + # Undump the object using Marshal.load, then + # unzip it using GZip.gunzip. + # + # The result is commonly a Tokens object, but + # this is not guaranteed. + def Tokens.load dump + dump = GZip.gunzip dump + @dump = Marshal.load dump + end + + alias text_token push + def begin_group kind; push :begin_group, kind end + def end_group kind; push :end_group, kind end + def begin_line kind; push :begin_line, kind end + def end_line kind; push :end_line, kind end + alias tokens concat + + end + +end |