diff options
Diffstat (limited to 'vendor/gems/coderay-0.9.7/lib/coderay/scanner.rb')
| -rw-r--r-- | vendor/gems/coderay-0.9.7/lib/coderay/scanner.rb | 298 |
1 files changed, 0 insertions, 298 deletions
diff --git a/vendor/gems/coderay-0.9.7/lib/coderay/scanner.rb b/vendor/gems/coderay-0.9.7/lib/coderay/scanner.rb deleted file mode 100644 index b12c865fb..000000000 --- a/vendor/gems/coderay-0.9.7/lib/coderay/scanner.rb +++ /dev/null @@ -1,298 +0,0 @@ -module CodeRay - - require 'coderay/helpers/plugin' - - # = Scanners - # - # This module holds the Scanner class and its subclasses. - # For example, the Ruby scanner is named CodeRay::Scanners::Ruby - # can be found in coderay/scanners/ruby. - # - # Scanner also provides methods and constants for the register - # mechanism and the [] method that returns the Scanner class - # belonging to the given lang. - # - # See PluginHost. - module Scanners - extend PluginHost - plugin_path File.dirname(__FILE__), 'scanners' - - require 'strscan' - - # = Scanner - # - # The base class for all Scanners. - # - # It is a subclass of Ruby's great +StringScanner+, which - # makes it easy to access the scanning methods inside. - # - # It is also +Enumerable+, so you can use it like an Array of - # Tokens: - # - # require 'coderay' - # - # c_scanner = CodeRay::Scanners[:c].new "if (*p == '{') nest++;" - # - # for text, kind in c_scanner - # puts text if kind == :operator - # end - # - # # prints: (*==)++; - # - # OK, this is a very simple example :) - # You can also use +map+, +any?+, +find+ and even +sort_by+, - # if you want. - class Scanner < StringScanner - - extend Plugin - plugin_host Scanners - - # Raised if a Scanner fails while scanning - ScanError = Class.new(Exception) - - require 'coderay/helpers/word_list' - - # The default options for all scanner classes. - # - # Define @default_options for subclasses. - DEFAULT_OPTIONS = { :stream => false } - - KINDS_NOT_LOC = [:comment, :doctype] - - class << self - - # Returns if the Scanner can be used in streaming mode. - def streamable? - is_a? Streamable - end - - def normify code - code = code.to_s - if code.respond_to?(:encoding) && (code.encoding.name != 'UTF-8' || !code.valid_encoding?) - code = code.dup - original_encoding = code.encoding - code.force_encoding 'Windows-1252' - unless code.valid_encoding? - code.force_encoding original_encoding - if code.encoding.name == 'UTF-8' - code.encode! 'UTF-16BE', :invalid => :replace, :undef => :replace, :replace => '?' - end - code.encode! 'UTF-8', :invalid => :replace, :undef => :replace, :replace => '?' - end - end - code.to_unix - end - - def file_extension extension = nil - if extension - @file_extension = extension.to_s - else - @file_extension ||= plugin_id.to_s - end - end - - end - -=begin -## Excluded for speed reasons; protected seems to make methods slow. - - # Save the StringScanner methods from being called. - # This would not be useful for highlighting. - strscan_public_methods = - StringScanner.instance_methods - - StringScanner.ancestors[1].instance_methods - protected(*strscan_public_methods) -=end - - # Create a new Scanner. - # - # * +code+ is the input String and is handled by the superclass - # StringScanner. - # * +options+ is a Hash with Symbols as keys. - # It is merged with the default options of the class (you can - # overwrite default options here.) - # * +block+ is the callback for streamed highlighting. - # - # If you set :stream to +true+ in the options, the Scanner uses a - # TokenStream with the +block+ as callback to handle the tokens. - # - # Else, a Tokens object is used. - def initialize code='', options = {}, &block - raise "I am only the basic Scanner class. I can't scan "\ - "anything. :( Use my subclasses." if self.class == Scanner - - @options = self.class::DEFAULT_OPTIONS.merge options - - super Scanner.normify(code) - - @tokens = options[:tokens] - if @options[:stream] - warn "warning in CodeRay::Scanner.new: :stream is set, "\ - "but no block was given" unless block_given? - raise NotStreamableError, self unless kind_of? Streamable - @tokens ||= TokenStream.new(&block) - else - warn "warning in CodeRay::Scanner.new: Block given, "\ - "but :stream is #{@options[:stream]}" if block_given? - @tokens ||= Tokens.new - end - @tokens.scanner = self - - setup - end - - def reset - super - reset_instance - end - - def string= code - code = Scanner.normify(code) - if defined?(RUBY_DESCRIPTION) && RUBY_DESCRIPTION['rubinius 1.0.1'] - reset_state - @string = code - else - super code - end - reset_instance - end - - # More mnemonic accessor name for the input string. - alias code string - alias code= string= - - # Returns the Plugin ID for this scanner. - def lang - self.class.plugin_id - end - - # Scans the code and returns all tokens in a Tokens object. - def tokenize new_string=nil, options = {} - options = @options.merge(options) - self.string = new_string if new_string - @cached_tokens = - if @options[:stream] # :stream must have been set already - reset unless new_string - scan_tokens @tokens, options - @tokens - else - scan_tokens @tokens, options - end - end - - def tokens - @cached_tokens ||= tokenize - end - - # Whether the scanner is in streaming mode. - def streaming? - !!@options[:stream] - end - - # Traverses the tokens. - def each &block - raise ArgumentError, - 'Cannot traverse TokenStream.' if @options[:stream] - tokens.each(&block) - end - include Enumerable - - # The current line position of the scanner. - # - # Beware, this is implemented inefficiently. It should be used - # for debugging only. - def line - string[0..pos].count("\n") + 1 - end - - def column pos = self.pos - return 0 if pos <= 0 - string = string() - if string.respond_to?(:bytesize) && (defined?(@bin_string) || string.bytesize != string.size) - @bin_string ||= string.dup.force_encoding('binary') - string = @bin_string - end - pos - (string.rindex(?\n, pos) || 0) - end - - def marshal_dump - @options - end - - def marshal_load options - @options = options - end - - protected - - # Can be implemented by subclasses to do some initialization - # that has to be done once per instance. - # - # Use reset for initialization that has to be done once per - # scan. - def setup - end - - # This is the central method, and commonly the only one a - # subclass implements. - # - # Subclasses must implement this method; it must return +tokens+ - # and must only use Tokens#<< for storing scanned tokens! - def scan_tokens tokens, options - raise NotImplementedError, - "#{self.class}#scan_tokens not implemented." - end - - def reset_instance - @tokens.clear unless @options[:keep_tokens] - @cached_tokens = nil - @bin_string = nil if defined? @bin_string - end - - # Scanner error with additional status information - def raise_inspect msg, tokens, state = 'No state given!', ambit = 30 - raise ScanError, <<-EOE % [ - - -***ERROR in %s: %s (after %d tokens) - -tokens: -%s - -current line: %d column: %d pos: %d -matched: %p state: %p -bol? = %p, eos? = %p - -surrounding code: -%p ~~ %p - - -***ERROR*** - - EOE - File.basename(caller[0]), - msg, - tokens.size, - tokens.last(10).map { |t| t.inspect }.join("\n"), - line, column, pos, - matched, state, bol?, eos?, - string[pos - ambit, ambit], - string[pos, ambit], - ] - end - - end - - end -end - -class String - # I love this hack. It seems to silence all dos/unix/mac newline problems. - def to_unix - if index ?\r - gsub(/\r\n?/, "\n") - else - self - end - end -end |