Commit 6c9c7dc5 authored by Eric Davis's avatar Eric Davis

Merge remote-tracking branch 'schmidt/f/290-bundler' into unstable

parents f273b7b1 ddcd4f28
......@@ -24,3 +24,6 @@
/vendor/rails
*.rbc
doc/app
/.bundle
/Gemfile.lock
/.rvmrc*
......@@ -27,3 +27,7 @@ vendor/rails
*.rbc
.svn/
.git/
doc/app
/.bundle
/Gemfile.lock
/.rvmrc*
source :rubygems
gem "rails", "2.3.11"
gem "coderay", "~> 0.9.7"
gem "i18n", "~> 0.4.2"
gem "rubytree", "~> 0.5.2", :require => 'tree'
group :test do
gem 'shoulda', '~> 2.10.3'
gem 'edavis10-object_daddy', :require => 'object_daddy'
gem 'mocha'
end
group :openid do
gem "ruby-openid", '~> 2.1.4', :require => 'openid'
end
group :rmagick do
gem "rmagick", "~> 1.15.17"
end
# Use the commented pure ruby gems, if you have not the needed prerequisites on
# board to compile the native ones. Note, that their use is discouraged, since
# their integration is propbably not that well tested and their are slower in
# orders of magnitude compared to their native counterparts. You have been
# warned.
#
platforms :mri do
group :mysql do
gem "mysql"
# gem "ruby-mysql"
end
group :mysql2 do
gem "mysql2", "~> 0.2.7"
end
group :postgres do
gem "pg", "~> 0.9.0"
# gem "postgres-pr"
end
group :sqlite do
gem "sqlite3-ruby", "< 1.3", :require => "sqlite3"
# please tell me, if you are fond of a pure ruby sqlite3 binding
end
end
platforms :jruby do
gem "jruby-openssl"
group :mysql do
gem "activerecord-jdbcmysql-adapter"
end
group :postgres do
gem "activerecord-jdbcpostgresql-adapter"
end
group :sqlite do
gem "activerecord-jdbcsqlite3-adapter"
end
end
# Load plugins' Gemfiles
Dir.glob File.expand_path("../vendor/plugins/*/Gemfile", __FILE__) do |file|
puts "Loading #{file} ..."
instance_eval File.read(file)
end
......@@ -36,6 +36,15 @@ module Rails
class Boot
def run
load_initializer
# This block was added for bundler support while following setup
# instructions from http://gembundler.com/rails23.html
Rails::Initializer.class_eval do
def load_gems
@bundler_loaded ||= Bundler.require :default, Rails.env
end
end
Rails::Initializer.run(:set_load_path)
end
end
......
......@@ -50,11 +50,8 @@ Rails::Initializer.run do |config|
# It will automatically turn deliveries on
config.action_mailer.perform_deliveries = false
config.gem 'rubytree', :lib => 'tree'
config.gem 'coderay', :version => '~>0.9.7'
# Load any local configuration that is kept out of source control
# (e.g. gems, patches).
# (e.g. patches).
if File.exists?(File.join(File.dirname(__FILE__), 'additional_environment.rb'))
instance_eval File.read(File.join(File.dirname(__FILE__), 'additional_environment.rb'))
end
......
......@@ -23,7 +23,3 @@ config.action_controller.session = {
# Skip protect_from_forgery in requests http://m.onkey.org/2007/9/28/csrf-protection-for-your-existing-rails-application
config.action_controller.allow_forgery_protection = false
config.gem "shoulda", :version => "~> 2.10.3"
config.gem "edavis10-object_daddy", :lib => "object_daddy"
config.gem "mocha"
begin
require "rubygems"
require "bundler"
rescue LoadError
raise "Could not load the bundler gem. Install it with `gem install bundler`."
end
if Gem::Version.new(Bundler::VERSION) <= Gem::Version.new("0.9.24")
raise RuntimeError, "Your bundler version is too old for Rails 2.3." +
"Run `gem install bundler` to upgrade."
end
begin
# Set up load paths for all bundled gems
ENV["BUNDLE_GEMFILE"] = File.expand_path("../../Gemfile", __FILE__)
Bundler.setup
rescue Bundler::GemNotFound
raise RuntimeError, "Bundler couldn't find some gems." +
"Did you run `bundle install`?"
end
--- !ruby/object:Gem::Specification
name: coderay
version: !ruby/object:Gem::Version
hash: 53
prerelease: false
segments:
- 0
- 9
- 7
version: 0.9.7
platform: ruby
authors:
- murphy
autorequire:
bindir: bin
cert_chain: []
date: 2011-01-15 00:00:00 +01:00
default_executable:
dependencies: []
description: |
Fast and easy syntax highlighting for selected languages, written in Ruby.
Comes with RedCloth integration and LOC counter.
email: murphy@rubychan.de
executables:
- coderay
- coderay_stylesheet
extensions: []
extra_rdoc_files:
- lib/README
- FOLDERS
files:
- ./lib/coderay/duo.rb
- ./lib/coderay/encoder.rb
- ./lib/coderay/encoders/_map.rb
- ./lib/coderay/encoders/comment_filter.rb
- ./lib/coderay/encoders/count.rb
- ./lib/coderay/encoders/debug.rb
- ./lib/coderay/encoders/div.rb
- ./lib/coderay/encoders/filter.rb
- ./lib/coderay/encoders/html/css.rb
- ./lib/coderay/encoders/html/numerization.rb
- ./lib/coderay/encoders/html/output.rb
- ./lib/coderay/encoders/html.rb
- ./lib/coderay/encoders/json.rb
- ./lib/coderay/encoders/lines_of_code.rb
- ./lib/coderay/encoders/null.rb
- ./lib/coderay/encoders/page.rb
- ./lib/coderay/encoders/span.rb
- ./lib/coderay/encoders/statistic.rb
- ./lib/coderay/encoders/term.rb
- ./lib/coderay/encoders/text.rb
- ./lib/coderay/encoders/token_class_filter.rb
- ./lib/coderay/encoders/xml.rb
- ./lib/coderay/encoders/yaml.rb
- ./lib/coderay/for_redcloth.rb
- ./lib/coderay/helpers/file_type.rb
- ./lib/coderay/helpers/gzip_simple.rb
- ./lib/coderay/helpers/plugin.rb
- ./lib/coderay/helpers/word_list.rb
- ./lib/coderay/scanner.rb
- ./lib/coderay/scanners/_map.rb
- ./lib/coderay/scanners/c.rb
- ./lib/coderay/scanners/cpp.rb
- ./lib/coderay/scanners/css.rb
- ./lib/coderay/scanners/debug.rb
- ./lib/coderay/scanners/delphi.rb
- ./lib/coderay/scanners/diff.rb
- ./lib/coderay/scanners/groovy.rb
- ./lib/coderay/scanners/html.rb
- ./lib/coderay/scanners/java/builtin_types.rb
- ./lib/coderay/scanners/java.rb
- ./lib/coderay/scanners/java_script-0.9.6.rb
- ./lib/coderay/scanners/java_script.rb
- ./lib/coderay/scanners/json.rb
- ./lib/coderay/scanners/nitro_xhtml.rb
- ./lib/coderay/scanners/php.rb
- ./lib/coderay/scanners/plaintext.rb
- ./lib/coderay/scanners/python.rb
- ./lib/coderay/scanners/rhtml.rb
- ./lib/coderay/scanners/ruby/patterns.rb
- ./lib/coderay/scanners/ruby.rb
- ./lib/coderay/scanners/scheme.rb
- ./lib/coderay/scanners/sql.rb
- ./lib/coderay/scanners/xml.rb
- ./lib/coderay/scanners/yaml.rb
- ./lib/coderay/style.rb
- ./lib/coderay/styles/_map.rb
- ./lib/coderay/styles/cycnus.rb
- ./lib/coderay/styles/murphy.rb
- ./lib/coderay/token_classes.rb
- ./lib/coderay/tokens.rb
- ./lib/coderay.rb
- ./Rakefile
- ./test/functional/basic.rb
- ./test/functional/basic.rbc
- ./test/functional/for_redcloth.rb
- ./test/functional/for_redcloth.rbc
- ./test/functional/load_plugin_scanner.rb
- ./test/functional/load_plugin_scanner.rbc
- ./test/functional/suite.rb
- ./test/functional/suite.rbc
- ./test/functional/vhdl.rb
- ./test/functional/vhdl.rbc
- ./test/functional/word_list.rb
- ./test/functional/word_list.rbc
- ./lib/README
- ./LICENSE
- lib/README
- FOLDERS
- bin/coderay
- bin/coderay_stylesheet
has_rdoc: true
homepage: http://coderay.rubychan.de
licenses: []
post_install_message:
rdoc_options:
- -SNw2
- -mlib/README
- -t CodeRay Documentation
require_paths:
- lib
required_ruby_version: !ruby/object:Gem::Requirement
none: false
requirements:
- - ">="
- !ruby/object:Gem::Version
hash: 51
segments:
- 1
- 8
- 2
version: 1.8.2
required_rubygems_version: !ruby/object:Gem::Requirement
none: false
requirements:
- - ">="
- !ruby/object:Gem::Version
hash: 3
segments:
- 0
version: "0"
requirements: []
rubyforge_project: coderay
rubygems_version: 1.3.7
signing_key:
specification_version: 3
summary: Fast syntax highlighting for selected languages.
test_files:
- ./test/functional/suite.rb
= CodeRay - Trunk folder structure
== bench - Benchmarking system
All benchmarking stuff goes here.
Test inputs are stored in files named <code>example.<lang></code>.
Test outputs go to <code>bench/test.<encoder-default-file-extension></code>.
Run <code>bench/bench.rb</code> to get a usage description.
Run <code>rake bench</code> to perform an example benchmark.
== bin - Scripts
Executional files for CodeRay.
== demo - Demos and functional tests
Demonstrational scripts to show of CodeRay's features.
Run them as functional tests with <code>rake test:demos</code>.
== etc - Lots of stuff
Some addidtional files for CodeRay, mainly graphics and Vim scripts.
== gem_server - Gem output folder
For <code>rake gem</code>.
== lib - CodeRay library code
This is the base directory for the CodeRay library.
== rake_helpers - Rake helper libraries
Some files to enhance Rake, including the Autumnal Rdoc template and some scripts.
== test - Tests
Tests for the scanners.
Each language has its own subfolder and sub-suite.
Run with <code>rake test</code>.
This diff is collapsed.
require 'rake/rdoctask'
ROOT = '.'
LIB_ROOT = File.join ROOT, 'lib'
EXTRA_RDOC_FILES = %w(lib/README FOLDERS)
task :default => :test
if File.directory? 'rake_tasks'
# load rake tasks from subfolder
for task_file in Dir['rake_tasks/*.rake'].sort
load task_file
end
else
# fallback tasks when rake_tasks folder is not present
desc 'Run CodeRay tests (basic)'
task :test do
ruby './test/functional/suite.rb'
ruby './test/functional/for_redcloth.rb'
end
desc 'Generate documentation for CodeRay'
Rake::RDocTask.new :doc do |rd|
rd.title = 'CodeRay Documentation'
rd.main = 'lib/README'
rd.rdoc_files.add Dir['lib']
rd.rdoc_files.add 'lib/README'
rd.rdoc_files.add 'FOLDERS'
rd.rdoc_dir = 'doc'
end
end
\ No newline at end of file
#!/usr/bin/env ruby
# CodeRay Executable
#
# Version: 0.2
# Author: murphy
require 'coderay'
if ARGV.empty?
$stderr.puts <<-USAGE
CodeRay #{CodeRay::VERSION} (http://coderay.rubychan.de)
Usage:
coderay file [-<format>]
coderay -<lang> [-<format>] [< file] [> output]
Defaults:
lang: based on file extension
format: ANSI colorized output for terminal, HTML page for files
Examples:
coderay foo.rb # colorized output to terminal, based on file extension
coderay foo.rb -loc # print LOC count, based on file extension and format
coderay foo.rb > foo.html # HTML page output to file, based on extension
coderay -ruby < foo.rb # colorized output to terminal, based on lang
coderay -ruby -loc < foo.rb # print LOC count, based on lang
coderay -ruby -page foo.rb # HTML page output to terminal, based on lang and format
coderay -ruby -page foo.rb > foo.html # HTML page output to file, based on lang and format
USAGE
end
first, second = ARGV
def read
file = ARGV.grep(/^(?!-)/).last
if file
if File.exist?(file)
File.read file
else
$stderr.puts "No such file: #{file}"
end
else
$stdin.read
end
end
if first
if first[/-(\w+)/] == first
lang = $1
input = read
tokens = :scan
else
file = first
unless File.exist? file
$stderr.puts "No such file: #{file}"
exit 2
end
tokens = CodeRay.scan_file file
end
else
$stderr.puts 'No lang/file given.'
exit 1
end
if second
if second[/-(\w+)/] == second
format = $1.to_sym
else
raise 'invalid format (must be -xxx)'
end
else
if $stdout.tty?
format = :term
else
$stderr.puts 'No format given; setting to default (HTML Page).'
format = :page
end
end
if tokens == :scan
output = CodeRay::Duo[lang => format].highlight input
else
output = tokens.encode format
end
out = $stdout
out.puts output
#!/usr/bin/env ruby
require 'coderay'
puts CodeRay::Encoders[:html]::CSS.new.stylesheet
= CodeRay
[- Tired of blue'n'gray? Try the original version of this documentation on
coderay.rubychan.de[http://coderay.rubychan.de/doc/] (use Ctrl+Click to open it in its own frame.) -]
== About
CodeRay is a Ruby library for syntax highlighting.
Syntax highlighting means: You put your code in, and you get it back colored;
Keywords, strings, floats, comments - all in different colors.
And with line numbers.
*Syntax* *Highlighting*...
* makes code easier to read and maintain
* lets you detect syntax errors faster
* helps you to understand the syntax of a language
* looks nice
* is what everybody should have on their website
* solves all your problems and makes the girls run after you
Version: 0.9.7
Author:: murphy (Kornelius Kalnbach)
Contact:: murphy rubychan de
Website:: coderay.rubychan.de[http://coderay.rubychan.de]
License:: GNU LGPL; see LICENSE file in the main directory.
== Installation
You need RubyGems[http://rubyforge.org/frs/?group_id=126].
% gem install coderay
=== Dependencies
CodeRay needs Ruby 1.8.6 or later. It also runs with Ruby 1.9.1+ and JRuby 1.1+.
== Example Usage
(Forgive me, but this is not highlighted.)
require 'coderay'
tokens = CodeRay.scan "puts 'Hello, world!'", :ruby
page = tokens.html :line_numbers => :inline, :wrap => :page
puts page
== Documentation
See CodeRay.
Please report errors in this documentation to <murphy rubychan de>.
== Credits
=== Special Thanks to
* licenser (Heinz N. Gies) for ending my QBasic career, inventing the Coder
project and the input/output plugin system.
CodeRay would not exist without him.
* bovi (Daniel Bovensiepen) for helping me out on various occasions.
=== Thanks to
* Caleb Clausen for writing RubyLexer (see
http://rubyforge.org/projects/rubylexer) and lots of very interesting mail
traffic
* birkenfeld (Georg Brandl) and mitsuhiku (Arnim Ronacher) for PyKleur, now pygments.
You guys rock!
* Jamis Buck for writing Syntax (see http://rubyforge.org/projects/syntax)
I got some useful ideas from it.
* Doug Kearns and everyone else who worked on ruby.vim - it not only helped me
coding CodeRay, but also gave me a wonderful target to reach for the Ruby
scanner.
* everyone who uses CodeBB on http://www.rubyforen.de and http://www.python-forum.de
* iGEL, magichisoka, manveru, WoNáDo and everyone I forgot from rubyforen.de
* Dethix from ruby-mine.de
* zickzackw
* Dookie (who is no longer with us...) and Leonidas from http://www.python-forum.de
* Andreas Schwarz for finding out that CaseIgnoringWordList was not case
ignoring! Such things really make you write tests.
* closure for the first version of the Scheme scanner.
* Stefan Walk for the first version of the JavaScript and PHP scanners.
* Josh Goebel for another version of the JavaScript scanner, a SQL and a Diff scanner.
* Jonathan Younger for pointing out the licence confusion caused by wrong LICENSE file.
* Jeremy Hinegardner for finding the shebang-on-empty-file bug in FileType.
* Charles Oliver Nutter and Yehuda Katz for helping me benchmark CodeRay on JRuby.
* Andreas Neuhaus for pointing out a markup bug in coderay/for_redcloth.
* 0xf30fc7 for the FileType patch concerning Delphi file extensions.
* The folks at redmine.org - thank you for using and fixing CodeRay!
* Keith Pitt for his SQL scanners
* Rob Aldred for the terminal encoder
* Trans for pointing out $DEBUG dependencies
* Flameeyes for finding that Term::ANSIColor was obsolete
* Etienne Massip for reporting a serious bug in JavaScript scanner
* matz and all Ruby gods and gurus
* The inventors of: the computer, the internet, the true color display, HTML &
CSS, VIM, Ruby, pizza, microwaves, guitars, scouting, programming, anime,
manga, coke and green ice tea.
Where would we be without all those people?
=== Created using
* Ruby[http://ruby-lang.org/]
* Chihiro (my Sony VAIO laptop); Henrietta (my old MacBook);
Triella, born Rico (my new MacBook); as well as
Seras and Hikari (my PCs)
* RDE[http://homepage2.nifty.com/sakazuki/rde_e.html],
VIM[http://vim.org] and TextMate[http://macromates.com]
* Subversion[http://subversion.tigris.org/]
* Redmine[http://redmine.org/]
* Firefox[http://www.mozilla.org/products/firefox/],
Firebug[http://getfirebug.com/], Safari[http://www.apple.com/safari/], and
Thunderbird[http://www.mozilla.org/products/thunderbird/]
* RubyGems[http://docs.rubygems.org/] and Rake[http://rake.rubyforge.org/]
* TortoiseSVN[http://tortoisesvn.tigris.org/] using Apache via
XAMPP[http://www.apachefriends.org/en/xampp.html]
* RDoc (though I'm quite unsatisfied with it)
* Microsoft Windows (yes, I confess!) and MacOS X
* GNUWin32, MinGW and some other tools to make the shell under windows a bit
less useless
* Term::ANSIColor[http://term-ansicolor.rubyforge.org/]
* PLEAC[http://pleac.sourceforge.net/] code examples
=== Free
* As you can see, CodeRay was created under heavy use of *free* software.
* So CodeRay is also *free*.
* If you use CodeRay to create software, think about making this software
*free*, too.
* Thanks :)
This diff is collapsed.
module CodeRay
# = Duo
#
# A Duo is a convenient way to use CodeRay. You just create a Duo,
# giving it a lang (language of the input code) and a format (desired
# output format), and call Duo#highlight with the code.
#
# Duo makes it easy to re-use both scanner and encoder for a repetitive
# task. It also provides a very easy interface syntax:
#
# require 'coderay'
# CodeRay::Duo[:python, :div].highlight 'import this'
#
# Until you want to do uncommon things with CodeRay, I recommend to use
# this method, since it takes care of everything.
class Duo
attr_accessor :lang, :format, :options
# Create a new Duo, holding a lang and a format to highlight code.
#
# simple:
# CodeRay::Duo[:ruby, :page].highlight 'bla 42'
#
# streaming:
# CodeRay::Duo[:ruby, :page].highlight 'bar 23', :stream => true
#
# with options:
# CodeRay::Duo[:ruby, :html, :hint => :debug].highlight '????::??'
#
# alternative syntax without options:
# CodeRay::Duo[:ruby => :statistic].encode 'class << self; end'
#
# alternative syntax with options:
# CodeRay::Duo[{ :ruby => :statistic }, :do => :something].encode 'abc'
#
# The options are forwarded to scanner and encoder
# (see CodeRay.get_scanner_options).
def initialize lang = nil, format = nil, options = {}
if format == nil and lang.is_a? Hash and lang.size == 1
@lang = lang.keys.first
@format = lang[@lang]
else
@lang = lang
@format = format
end
@options = options
end
class << self
# To allow calls like Duo[:ruby, :html].highlight.
alias [] new
end
# The scanner of the duo. Only created once.
def scanner
@scanner ||= CodeRay.scanner @lang, CodeRay.get_scanner_options(@options)
end
# The encoder of the duo. Only created once.
def encoder
@encoder ||= CodeRay.encoder @format, @options
end
# Tokenize and highlight the code using +scanner+ and +encoder+.
#
# If the :stream option is set, the Duo will go into streaming mode,
# saving memory for the cost of time.
def encode code, options = { :stream => false }
stream = options.delete :stream
options = @options.merge options
if stream
encoder.encode_stream(code, @lang, options)
else
scanner.code = code
encoder.encode_tokens(scanner.tokenize, options)
end
end
alias highlight encode
end
end
module CodeRay
# This module holds the Encoder class and its subclasses.
# For example, the HTML encoder is named CodeRay::Encoders::HTML
# can be found in coderay/encoders/html.
#
# Encoders also provides methods and constants for the register
# mechanism and the [] method that returns the Encoder class
# belonging to the given format.
module Encoders
extend PluginHost
plugin_path File.dirname(__FILE__), 'encoders'
# = Encoder
#
# The Encoder base class. Together with Scanner and
# Tokens, it forms the highlighting triad.
#
# Encoder instances take a Tokens object and do something with it.
#
# The most common Encoder is surely the HTML encoder
# (CodeRay::Encoders::HTML). It highlights the code in a colorful
# html page.
# If you want the highlighted code in a div or a span instead,
# use its subclasses Div and Span.
class Encoder
extend Plugin
plugin_host Encoders
attr_reader :token_stream
class << self
# Returns if the Encoder can be used in streaming mode.
def streamable?
is_a? Streamable
end
# If FILE_EXTENSION isn't defined, this method returns the
# downcase class name instead.
def const_missing sym
if sym == :FILE_EXTENSION
plugin_id
else
super
end
end
end
# Subclasses are to store their default options in this constant.
DEFAULT_OPTIONS = { :stream => false }
# The options you gave the Encoder at creating.
attr_accessor :options
# Creates a new Encoder.
# +options+ is saved and used for all encode operations, as long
# as you don't overwrite it there by passing additional options.
#
# Encoder objects provide three encode methods:
# - encode simply takes a +code+ string and a +lang+
# - encode_tokens expects a +tokens+ object instead
# - encode_stream is like encode, but uses streaming mode.
#
# Each method has an optional +options+ parameter. These are
# added to the options you passed at creation.
def initialize options = {}
@options = self.class::DEFAULT_OPTIONS.merge options
raise "I am only the basic Encoder class. I can't encode "\
"anything. :( Use my subclasses." if self.class == Encoder
end
# Encode a Tokens object.
def encode_tokens tokens, options = {}
options = @options.merge options
setup options
compile tokens, options
finish options
end
# Encode the given +code+ after tokenizing it using the Scanner
# for +lang+.
def encode code, lang, options = {}
options = @options.merge options
scanner_options = CodeRay.get_scanner_options(options)
tokens = CodeRay.scan code, lang, scanner_options
encode_tokens tokens, options
end
# You can use highlight instead of encode, if that seems
# more clear to you.
alias highlight encode
# Encode the given +code+ using the Scanner for +lang+ in
# streaming mode.
def encode_stream code, lang, options = {}
raise NotStreamableError, self unless kind_of? Streamable
options = @options.merge options
setup options
scanner_options = CodeRay.get_scanner_options options
@token_stream =
CodeRay.scan_stream code, lang, scanner_options, &self
finish options
end
# Behave like a proc. The token method is converted to a proc.
def to_proc
method(:token).to_proc
end
# Return the default file extension for outputs of this encoder.
def file_extension
self.class::FILE_EXTENSION
end
protected
# Called with merged options before encoding starts.
# Sets @out to an empty string.
#
# See the HTML Encoder for an example of option caching.
def setup options
@out = ''
end
# Called with +content+ and +kind+ of the currently scanned token.
# For simple scanners, it's enougth to implement this method.
#
# By default, it calls text_token or block_token, depending on
# whether +content+ is a String.
def token content, kind
encoded_token =
if content.is_a? ::String
text_token content, kind
elsif content.is_a? ::Symbol
block_token content, kind
else
raise 'Unknown token content type: %p' % [content]
end
append_encoded_token_to_output encoded_token
end
def append_encoded_token_to_output encoded_token
@out << encoded_token if encoded_token && defined?(@out) && @out
end
# Called for each text token ([text, kind]), where text is a String.
def text_token text, kind
end
# Called for each block (non-text) token ([action, kind]),
# where +action+ is a Symbol.
#
# Calls open_token, close_token, begin_line, and end_line according to
# the value of +action+.
def block_token action, kind
case action
when :open
open_token kind
when :close
close_token kind
when :begin_line
begin_line kind
when :end_line
end_line kind
else
raise 'unknown block action: %p' % action
end
end
# Called for each block token at the start of the block ([:open, kind]).
def open_token kind
end
# Called for each block token end of the block ([:close, kind]).
def close_token kind
end
# Called for each line token block at the start of the line ([:begin_line, kind]).
def begin_line kind
end
# Called for each line token block at the end of the line ([:end_line, kind]).
def end_line kind
end
# Called with merged options after encoding starts.
# The return value is the result of encoding, typically @out.
def finish options
@out
end
# Do the encoding.
#
# The already created +tokens+ object must be used; it can be a
# TokenStream or a Tokens object.
if RUBY_VERSION >= '1.9'
def compile tokens, options
for text, kind in tokens
token text, kind
end
end
else
def compile tokens, options
tokens.each(&self)
end
end
end
end
end
module CodeRay
module Encoders
map \
:loc => :lines_of_code,
:plain => :text,
:stats => :statistic,
:terminal => :term,
:tex => :latex
end
end
($:.unshift '../..'; require 'coderay') unless defined? CodeRay
module CodeRay
module Encoders
load :token_class_filter
class CommentFilter < TokenClassFilter
register_for :comment_filter
DEFAULT_OPTIONS = superclass::DEFAULT_OPTIONS.merge \
:exclude => [:comment]
end
end
end
if $0 == __FILE__
$VERBOSE = true
$: << File.join(File.dirname(__FILE__), '..')
eval DATA.read, nil, $0, __LINE__ + 4
end
__END__
require 'test/unit'
class CommentFilterTest < Test::Unit::TestCase
def test_filtering_comments
tokens = CodeRay.scan <<-RUBY, :ruby
#!/usr/bin/env ruby
# a minimal Ruby program
puts "Hello world!"
RUBY
assert_equal <<-RUBY_FILTERED, tokens.comment_filter.text
#!/usr/bin/env ruby
puts "Hello world!"
RUBY_FILTERED
end
end
\ No newline at end of file
module CodeRay
module Encoders
class Count < Encoder
include Streamable
register_for :count
protected
def setup options
@out = 0
end
def token text, kind
@out += 1
end
end
end
end
module CodeRay
module Encoders
# = Debug Encoder
#
# Fast encoder producing simple debug output.
#
# It is readable and diff-able and is used for testing.
#
# You cannot fully restore the tokens information from the
# output, because consecutive :space tokens are merged.
# Use Tokens#dump for caching purposes.
class Debug < Encoder
include Streamable
register_for :debug
FILE_EXTENSION = 'raydebug'
protected
def text_token text, kind
if kind == :space
text
else
text = text.gsub(/[)\\]/, '\\\\\0') # escape ) and \
"#{kind}(#{text})"
end
end
def open_token kind
"#{kind}<"
end
def close_token kind
">"
end
def begin_line kind
"#{kind}["
end
def end_line kind
"]"
end
end
end
end
module CodeRay
module Encoders
load :html
class Div < HTML
FILE_EXTENSION = 'div.html'
register_for :div
DEFAULT_OPTIONS = HTML::DEFAULT_OPTIONS.merge \
:css => :style,
:wrap => :div
end
end
end
($:.unshift '../..'; require 'coderay') unless defined? CodeRay
module CodeRay
module Encoders
class Filter < Encoder
register_for :filter
protected
def setup options
@out = Tokens.new
end
def text_token text, kind
[text, kind] if include_text_token? text, kind
end
def include_text_token? text, kind
true
end
def block_token action, kind
[action, kind] if include_block_token? action, kind
end
def include_block_token? action, kind
true
end
end
end
end
if $0 == __FILE__
$VERBOSE = true
$: << File.join(File.dirname(__FILE__), '..')
eval DATA.read, nil, $0, __LINE__ + 4
end
__END__
require 'test/unit'
class FilterTest < Test::Unit::TestCase
def test_creation
assert CodeRay::Encoders::Filter < CodeRay::Encoders::Encoder
filter = nil
assert_nothing_raised do
filter = CodeRay.encoder :filter
end
assert_kind_of CodeRay::Encoders::Encoder, filter
end
def test_filtering_text_tokens
tokens = CodeRay::Tokens.new
10.times do |i|
tokens << [i.to_s, :index]
end
assert_equal tokens, CodeRay::Encoders::Filter.new.encode_tokens(tokens)
assert_equal tokens, tokens.filter
end
def test_filtering_block_tokens
tokens = CodeRay::Tokens.new
10.times do |i|
tokens << [:open, :index]
tokens << [i.to_s, :content]
tokens << [:close, :index]
end
assert_equal tokens, CodeRay::Encoders::Filter.new.encode_tokens(tokens)
assert_equal tokens, tokens.filter
end
end
require 'set'
module CodeRay
module Encoders
# = HTML Encoder
#
# This is CodeRay's most important highlighter:
# It provides save, fast XHTML generation and CSS support.
#
# == Usage
#
# require 'coderay'
# puts CodeRay.scan('Some /code/', :ruby).html #-> a HTML page
# puts CodeRay.scan('Some /code/', :ruby).html(:wrap => :span)
# #-> <span class="CodeRay"><span class="co">Some</span> /code/</span>
# puts CodeRay.scan('Some /code/', :ruby).span #-> the same
#
# puts CodeRay.scan('Some code', :ruby).html(
# :wrap => nil,
# :line_numbers => :inline,
# :css => :style
# )
# #-> <span class="no">1</span> <span style="color:#036; font-weight:bold;">Some</span> code
#
# == Options
#
# === :tab_width
# Convert \t characters to +n+ spaces (a number.)
# Default: 8
#
# === :css
# How to include the styles; can be :class or :style.
#
# Default: :class
#
# === :wrap
# Wrap in :page, :div, :span or nil.
#
# You can also use Encoders::Div and Encoders::Span.
#
# Default: nil
#
# === :title
#
# The title of the HTML page (works only when :wrap is set to :page.)
#
# Default: 'CodeRay output'
#
# === :line_numbers
# Include line numbers in :table, :inline, :list or nil (no line numbers)
#
# Default: nil
#
# === :line_number_start
# Where to start with line number counting.
#
# Default: 1
#
# === :bold_every
# Make every +n+-th number appear bold.
#
# Default: 10
#
# === :highlight_lines
#
# Highlights certain line numbers.
# Can be any Enumerable, typically just an Array or Range, of numbers.
#
# Bolding is deactivated when :highlight_lines is set. It only makes sense
# in combination with :line_numbers.
#
# Default: nil
#
# === :hint
# Include some information into the output using the title attribute.
# Can be :info (show token type on mouse-over), :info_long (with full path)
# or :debug (via inspect).
#
# Default: false
class HTML < Encoder
include Streamable
register_for :html
FILE_EXTENSION = 'html'
DEFAULT_OPTIONS = {
:tab_width => 8,
:css => :class,
:style => :cycnus,
:wrap => nil,
:title => 'CodeRay output',
:line_numbers => nil,
:line_number_start => 1,
:bold_every => 10,
:highlight_lines => nil,
:hint => false,
}
helper :output, :css
attr_reader :css
protected
HTML_ESCAPE = { #:nodoc:
'&' => '&amp;',
'"' => '&quot;',
'>' => '&gt;',
'<' => '&lt;',
}
# This was to prevent illegal HTML.
# Strange chars should still be avoided in codes.
evil_chars = Array(0x00...0x20) - [?\n, ?\t, ?\s]
evil_chars.each { |i| HTML_ESCAPE[i.chr] = ' ' }
#ansi_chars = Array(0x7f..0xff)
#ansi_chars.each { |i| HTML_ESCAPE[i.chr] = '&#%d;' % i }
# \x9 (\t) and \xA (\n) not included
#HTML_ESCAPE_PATTERN = /[\t&"><\0-\x8\xB-\x1f\x7f-\xff]/
HTML_ESCAPE_PATTERN = /[\t"&><\0-\x8\xB-\x1f]/
TOKEN_KIND_TO_INFO = Hash.new { |h, kind|
h[kind] =
case kind
when :pre_constant
'Predefined constant'
else
kind.to_s.gsub(/_/, ' ').gsub(/\b\w/) { $&.capitalize }
end
}
TRANSPARENT_TOKEN_KINDS = [
:delimiter, :modifier, :content, :escape, :inline_delimiter,
].to_set
# Generate a hint about the given +classes+ in a +hint+ style.
#
# +hint+ may be :info, :info_long or :debug.
def self.token_path_to_hint hint, classes
title =
case hint
when :info
TOKEN_KIND_TO_INFO[classes.first]
when :info_long
classes.reverse.map { |kind| TOKEN_KIND_TO_INFO[kind] }.join('/')
when :debug
classes.inspect
end
title ? " title=\"#{title}\"" : ''
end
def setup options
super
@HTML_ESCAPE = HTML_ESCAPE.dup
@HTML_ESCAPE["\t"] = ' ' * options[:tab_width]
@opened = [nil]
@css = CSS.new options[:style]
hint = options[:hint]
if hint and not [:debug, :info, :info_long].include? hint
raise ArgumentError, "Unknown value %p for :hint; \
expected :info, :debug, false, or nil." % hint
end
case options[:css]
when :class
@css_style = Hash.new do |h, k|
c = CodeRay::Tokens::ClassOfKind[k.first]
if c == :NO_HIGHLIGHT and not hint
h[k.dup] = false
else
title = if hint
HTML.token_path_to_hint(hint, k[1..-1] << k.first)
else
''
end
if c == :NO_HIGHLIGHT
h[k.dup] = '<span%s>' % [title]
else
h[k.dup] = '<span%s class="%s">' % [title, c]
end
end
end
when :style
@css_style = Hash.new do |h, k|
if k.is_a? ::Array
styles = k.dup
else
styles = [k]
end
type = styles.first
classes = styles.map { |c| Tokens::ClassOfKind[c] }
if classes.first == :NO_HIGHLIGHT and not hint
h[k] = false
else
styles.shift if TRANSPARENT_TOKEN_KINDS.include? styles.first
title = HTML.token_path_to_hint hint, styles
style = @css[*classes]
h[k] =
if style
'<span%s style="%s">' % [title, style]
else
false
end
end
end
else
raise ArgumentError, "Unknown value %p for :css." % options[:css]
end
end
def finish options
not_needed = @opened.shift
@out << '</span>' * @opened.size
unless @opened.empty?
warn '%d tokens still open: %p' % [@opened.size, @opened]
end
@out.extend Output
@out.css = @css
@out.numerize! options[:line_numbers], options
@out.wrap! options[:wrap]
@out.apply_title! options[:title]
super
end
def token text, type = :plain
case text
when nil
# raise 'Token with nil as text was given: %p' % [[text, type]]
when String
if text =~ /#{HTML_ESCAPE_PATTERN}/o
text = text.gsub(/#{HTML_ESCAPE_PATTERN}/o) { |m| @HTML_ESCAPE[m] }
end
@opened[0] = type
if text != "\n" && style = @css_style[@opened]
@out << style << text << '</span>'
else
@out << text
end
# token groups, eg. strings
when :open
@opened[0] = type
@out << (@css_style[@opened] || '<span>')
@opened << type
when :close
if @opened.empty?
# nothing to close
else
if $CODERAY_DEBUG and (@opened.size == 1 or @opened.last != type)
raise 'Malformed token stream: Trying to close a token (%p) \
that is not open. Open are: %p.' % [type, @opened[1..-1]]
end
@out << '</span>'
@opened.pop
end
# whole lines to be highlighted, eg. a deleted line in a diff
when :begin_line
@opened[0] = type
if style = @css_style[@opened]
if style['class="']
@out << style.sub('class="', 'class="line ')
else
@out << style.sub('>', ' class="line">')
end
else
@out << '<span class="line">'
end
@opened << type
when :end_line
if @opened.empty?
# nothing to close
else
if $CODERAY_DEBUG and (@opened.size == 1 or @opened.last != type)
raise 'Malformed token stream: Trying to close a line (%p) \
that is not open. Open are: %p.' % [type, @opened[1..-1]]
end
@out << '</span>'
@opened.pop
end
else
raise 'unknown token kind: %p' % [text]
end
end
end
end
end
module CodeRay
module Encoders
class HTML
class CSS
attr :stylesheet
def CSS.load_stylesheet style = nil
CodeRay::Styles[style]
end
def initialize style = :default
@classes = Hash.new
style = CSS.load_stylesheet style
@stylesheet = [
style::CSS_MAIN_STYLES,
style::TOKEN_COLORS.gsub(/^(?!$)/, '.CodeRay ')
].join("\n")
parse style::TOKEN_COLORS
end
def [] *styles
cl = @classes[styles.first]
return '' unless cl
style = ''
1.upto(styles.size) do |offset|
break if style = cl[styles[offset .. -1]]
end
# warn 'Style not found: %p' % [styles] if style.empty?
return style
end
private
CSS_CLASS_PATTERN = /
( # $1 = selectors
(?:
(?: \s* \. [-\w]+ )+
\s* ,?
)+
)
\s* \{ \s*
( [^\}]+ )? # $2 = style
\s* \} \s*
|
( . ) # $3 = error
/mx
def parse stylesheet
stylesheet.scan CSS_CLASS_PATTERN do |selectors, style, error|
raise "CSS parse error: '#{error.inspect}' not recognized" if error
for selector in selectors.split(',')
classes = selector.scan(/[-\w]+/)
cl = classes.pop
@classes[cl] ||= Hash.new
@classes[cl][classes] = style.to_s.strip.delete(' ').chomp(';')
end
end
end
end
end
end
end
if $0 == __FILE__
require 'pp'
pp CodeRay::Encoders::HTML::CSS.new
end
module CodeRay
module Encoders
class HTML
module Output
def numerize *args
clone.numerize!(*args)
end
=begin NUMERIZABLE_WRAPPINGS = {
:table => [:div, :page, nil],
:inline => :all,
:list => [:div, :page, nil]
}
NUMERIZABLE_WRAPPINGS.default = :all
=end
def numerize! mode = :table, options = {}
return self unless mode
options = DEFAULT_OPTIONS.merge options
start = options[:line_number_start]
unless start.is_a? Integer
raise ArgumentError, "Invalid value %p for :line_number_start; Integer expected." % start
end
#allowed_wrappings = NUMERIZABLE_WRAPPINGS[mode]
#unless allowed_wrappings == :all or allowed_wrappings.include? options[:wrap]
# raise ArgumentError, "Can't numerize, :wrap must be in %p, but is %p" % [NUMERIZABLE_WRAPPINGS, options[:wrap]]
#end
bold_every = options[:bold_every]
highlight_lines = options[:highlight_lines]
bolding =
if bold_every == false && highlight_lines == nil
proc { |line| line.to_s }
elsif highlight_lines.is_a? Enumerable
highlight_lines = highlight_lines.to_set
proc do |line|
if highlight_lines.include? line
"<strong class=\"highlighted\">#{line}</strong>" # highlighted line numbers in bold
else
line.to_s
end
end
elsif bold_every.is_a? Integer
raise ArgumentError, ":bolding can't be 0." if bold_every == 0
proc do |line|
if line % bold_every == 0
"<strong>#{line}</strong>" # every bold_every-th number in bold
else
line.to_s
end
end
else
raise ArgumentError, 'Invalid value %p for :bolding; false or Integer expected.' % bold_every
end
case mode
when :inline
max_width = (start + line_count).to_s.size
line_number = start
gsub!(/^/) do
line_number_text = bolding.call line_number
indent = ' ' * (max_width - line_number.to_s.size) # TODO: Optimize (10^x)
res = "<span class=\"no\">#{indent}#{line_number_text}</span> "
line_number += 1
res
end
when :table
# This is really ugly.
# Because even monospace fonts seem to have different heights when bold,
# I make the newline bold, both in the code and the line numbers.
# FIXME Still not working perfect for Mr. Internet Exploder
line_numbers = (start ... start + line_count).to_a.map(&bolding).join("\n")
line_numbers << "\n" # also for Mr. MS Internet Exploder :-/
line_numbers.gsub!(/\n/) { "<tt>\n</tt>" }
line_numbers_table_tpl = TABLE.apply('LINE_NUMBERS', line_numbers)
gsub!("</div>\n", '</div>')
gsub!("\n", "<tt>\n</tt>")
wrap_in! line_numbers_table_tpl
@wrapped_in = :div
when :list
opened_tags = []
gsub!(/^.*$\n?/) do |line|
line.chomp!
open = opened_tags.join
line.scan(%r!<(/)?span[^>]*>?!) do |close,|
if close
opened_tags.pop
else
opened_tags << $&
end
end
close = '</span>' * opened_tags.size
"<li>#{open}#{line}#{close}</li>\n"
end
chomp!("\n")
wrap_in! LIST
@wrapped_in = :div
else
raise ArgumentError, 'Unknown value %p for mode: expected one of %p' %
[mode, [:table, :list, :inline]]
end
self
end
def line_count
line_count = count("\n")
position_of_last_newline = rindex(?\n)
if position_of_last_newline
after_last_newline = self[position_of_last_newline + 1 .. -1]
ends_with_newline = after_last_newline[/\A(?:<\/span>)*\z/]
line_count += 1 if not ends_with_newline
end
line_count
end
end
end
end
end
module CodeRay
module Encoders
class HTML
# This module is included in the output String from thew HTML Encoder.
#
# It provides methods like wrap, div, page etc.
#
# Remember to use #clone instead of #dup to keep the modules the object was
# extended with.
#
# TODO: more doc.
module Output
require 'coderay/encoders/html/numerization.rb'
attr_accessor :css
class << self
# This makes Output look like a class.
#
# Example:
#
# a = Output.new '<span class="co">Code</span>'
# a.wrap! :page
def new string, css = CSS.new, element = nil
output = string.clone.extend self
output.wrapped_in = element
output.css = css
output
end
# Raises an exception if an object that doesn't respond to to_str is extended by Output,
# to prevent users from misuse. Use Module#remove_method to disable.
def extended o
warn "The Output module is intended to extend instances of String, not #{o.class}." unless o.respond_to? :to_str
end
def make_stylesheet css, in_tag = false
sheet = css.stylesheet
sheet = <<-CSS if in_tag
<style type="text/css">
#{sheet}
</style>
CSS
sheet
end
def page_template_for_css css
sheet = make_stylesheet css
PAGE.apply 'CSS', sheet
end
# Define a new wrapper. This is meta programming.
def wrapper *wrappers
wrappers.each do |wrapper|
define_method wrapper do |*args|
wrap wrapper, *args
end
define_method "#{wrapper}!".to_sym do |*args|
wrap! wrapper, *args
end
end
end
end
wrapper :div, :span, :page
def wrapped_in? element
wrapped_in == element
end
def wrapped_in
@wrapped_in ||= nil
end
attr_writer :wrapped_in
def wrap_in template
clone.wrap_in! template
end
def wrap_in! template
Template.wrap! self, template, 'CONTENT'
self
end
def apply_title! title
self.sub!(/(<title>)(<\/title>)/) { $1 + title + $2 }
self
end
def wrap! element, *args
return self if not element or element == wrapped_in
case element
when :div
raise "Can't wrap %p in %p" % [wrapped_in, element] unless wrapped_in? nil
wrap_in! DIV
when :span
raise "Can't wrap %p in %p" % [wrapped_in, element] unless wrapped_in? nil
wrap_in! SPAN
when :page
wrap! :div if wrapped_in? nil
raise "Can't wrap %p in %p" % [wrapped_in, element] unless wrapped_in? :div
wrap_in! Output.page_template_for_css(@css)
if args.first.is_a?(Hash) && title = args.first[:title]
apply_title! title
end
self
when nil
return self
else
raise "Unknown value %p for :wrap" % element
end
@wrapped_in = element
self
end
def wrap *args
clone.wrap!(*args)
end
def stylesheet in_tag = false
Output.make_stylesheet @css, in_tag
end
class Template < String
def self.wrap! str, template, target
target = Regexp.new(Regexp.escape("<%#{target}%>"))
if template =~ target
str[0,0] = $`
str << $'
else
raise "Template target <%%%p%%> not found" % target
end
end
def apply target, replacement
target = Regexp.new(Regexp.escape("<%#{target}%>"))
if self =~ target
Template.new($` + replacement + $')
else
raise "Template target <%%%p%%> not found" % target
end
end
module Simple
def ` str #` <-- for stupid editors
Template.new str
end
end
end
extend Template::Simple
#-- don't include the templates in docu
SPAN = `<span class="CodeRay"><%CONTENT%></span>`
DIV = <<-`DIV`
<div class="CodeRay">
<div class="code"><pre><%CONTENT%></pre></div>
</div>
DIV
TABLE = <<-`TABLE`
<table class="CodeRay"><tr>
<td class="line_numbers" title="click to toggle" onclick="with (this.firstChild.style) { display = (display == '') ? 'none' : '' }"><pre><%LINE_NUMBERS%></pre></td>
<td class="code"><pre ondblclick="with (this.style) { overflow = (overflow == 'auto' || overflow == '') ? 'visible' : 'auto' }"><%CONTENT%></pre></td>
</tr></table>
TABLE
# title="double click to expand"
LIST = <<-`LIST`
<ol class="CodeRay">
<%CONTENT%>
</ol>
LIST
PAGE = <<-`PAGE`
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
"http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="de">
<head>
<meta http-equiv="content-type" content="text/html; charset=utf-8" />
<title></title>
<style type="text/css">
<%CSS%>
</style>
</head>
<body style="background-color: white;">
<%CONTENT%>
</body>
</html>
PAGE
end
end
end
end
($:.unshift '../..'; require 'coderay') unless defined? CodeRay
module CodeRay
module Encoders
# = JSON Encoder
class JSON < Encoder
register_for :json
FILE_EXTENSION = 'json'
protected
def setup options
begin
require 'json'
rescue LoadError
require 'rubygems'
require 'json'
end
@out = []
end
def text_token text, kind
{ :type => 'text', :text => text, :kind => kind }
end
def block_token action, kind
{ :type => 'block', :action => action, :kind => kind }
end
def finish options
@out.to_json
end
end
end
end
if $0 == __FILE__
$VERBOSE = true
$: << File.join(File.dirname(__FILE__), '..')
eval DATA.read, nil, $0, __LINE__ + 4
end
__END__
require 'test/unit'
$:.delete '.'
require 'rubygems' if RUBY_VERSION < '1.9'
class JSONEncoderTest < Test::Unit::TestCase
def test_json_output
tokens = CodeRay.scan <<-RUBY, :ruby
puts "Hello world!"
RUBY
require 'json'
assert_equal [
{"type"=>"text", "text"=>"puts", "kind"=>"ident"},
{"type"=>"text", "text"=>" ", "kind"=>"space"},
{"type"=>"block", "action"=>"open", "kind"=>"string"},
{"type"=>"text", "text"=>"\"", "kind"=>"delimiter"},
{"type"=>"text", "text"=>"Hello world!", "kind"=>"content"},
{"type"=>"text", "text"=>"\"", "kind"=>"delimiter"},
{"type"=>"block", "action"=>"close", "kind"=>"string"},
{"type"=>"text", "text"=>"\n", "kind"=>"space"}
], JSON.load(tokens.json)
end
end
\ No newline at end of file
($:.unshift '../..'; require 'coderay') unless defined? CodeRay
module CodeRay
module Encoders
# Counts the LoC (Lines of Code). Returns an Integer >= 0.
#
# Alias: :loc
#
# Everything that is not comment, markup, doctype/shebang, or an empty line,
# is considered to be code.
#
# For example,
# * HTML files not containing JavaScript have 0 LoC
# * in a Java class without comments, LoC is the number of non-empty lines
#
# A Scanner class should define the token kinds that are not code in the
# KINDS_NOT_LOC constant, which defaults to [:comment, :doctype].
class LinesOfCode < Encoder
register_for :lines_of_code
NON_EMPTY_LINE = /^\s*\S.*$/
def compile tokens, options
if scanner = tokens.scanner
kinds_not_loc = scanner.class::KINDS_NOT_LOC
else
warn ArgumentError, 'Tokens have no scanner.' if $VERBOSE
kinds_not_loc = CodeRay::Scanners::Scanner::KINDS_NOT_LOC
end
code = tokens.token_class_filter :exclude => kinds_not_loc
@loc = code.text.scan(NON_EMPTY_LINE).size
end
def finish options
@loc
end
end
end
end
if $0 == __FILE__
$VERBOSE = true
$: << File.join(File.dirname(__FILE__), '..')
eval DATA.read, nil, $0, __LINE__ + 4
end
__END__
require 'test/unit'
class LinesOfCodeTest < Test::Unit::TestCase
def test_creation
assert CodeRay::Encoders::LinesOfCode < CodeRay::Encoders::Encoder
filter = nil
assert_nothing_raised do
filter = CodeRay.encoder :loc
end
assert_kind_of CodeRay::Encoders::LinesOfCode, filter
assert_nothing_raised do
filter = CodeRay.encoder :lines_of_code
end
assert_kind_of CodeRay::Encoders::LinesOfCode, filter
end
def test_lines_of_code
tokens = CodeRay.scan <<-RUBY, :ruby
#!/usr/bin/env ruby
# a minimal Ruby program
puts "Hello world!"
RUBY
assert_equal 1, CodeRay::Encoders::LinesOfCode.new.encode_tokens(tokens)
assert_equal 1, tokens.lines_of_code
assert_equal 1, tokens.loc
end
def test_filtering_block_tokens
tokens = CodeRay::Tokens.new
tokens << ["Hello\n", :world]
tokens << ["Hello\n", :space]
tokens << ["Hello\n", :comment]
assert_equal 2, CodeRay::Encoders::LinesOfCode.new.encode_tokens(tokens)
assert_equal 2, tokens.lines_of_code
assert_equal 2, tokens.loc
end
end
\ No newline at end of file
module CodeRay
module Encoders
# = Null Encoder
#
# Does nothing and returns an empty string.
class Null < Encoder
include Streamable
register_for :null
# Defined for faster processing
def to_proc
proc {}
end
protected
def token(*)
# do nothing
end
end
end
end
module CodeRay
module Encoders
load :html
class Page < HTML
FILE_EXTENSION = 'html'
register_for :page
DEFAULT_OPTIONS = HTML::DEFAULT_OPTIONS.merge \
:css => :class,
:wrap => :page,
:line_numbers => :table
end
end
end
module CodeRay
module Encoders
load :html
class Span < HTML
FILE_EXTENSION = 'span.html'
register_for :span
DEFAULT_OPTIONS = HTML::DEFAULT_OPTIONS.merge \
:css => :style,
:wrap => :span
end
end
end
module CodeRay
module Encoders
# Makes a statistic for the given tokens.
class Statistic < Encoder
include Streamable
register_for :stats, :statistic
attr_reader :type_stats, :real_token_count
protected
TypeStats = Struct.new :count, :size
def setup options
@type_stats = Hash.new { |h, k| h[k] = TypeStats.new 0, 0 }
@real_token_count = 0
end
def generate tokens, options
@tokens = tokens
super
end
def text_token text, kind
@real_token_count += 1 unless kind == :space
@type_stats[kind].count += 1
@type_stats[kind].size += text.size
@type_stats['TOTAL'].size += text.size
@type_stats['TOTAL'].count += 1
end
# TODO Hierarchy handling
def block_token action, kind
@type_stats['TOTAL'].count += 1
@type_stats['open/close'].count += 1
end
STATS = <<-STATS
Code Statistics
Tokens %8d
Non-Whitespace %8d
Bytes Total %8d
Token Types (%d):
type count ratio size (average)
-------------------------------------------------------------
%s
STATS
# space 12007 33.81 % 1.7
TOKEN_TYPES_ROW = <<-TKR
%-20s %8d %6.2f %% %5.1f
TKR
def finish options
all = @type_stats['TOTAL']
all_count, all_size = all.count, all.size
@type_stats.each do |type, stat|
stat.size /= stat.count.to_f
end
types_stats = @type_stats.sort_by { |k, v| [-v.count, k.to_s] }.map do |k, v|
TOKEN_TYPES_ROW % [k, v.count, 100.0 * v.count / all_count, v.size]
end.join
STATS % [
all_count, @real_token_count, all_size,
@type_stats.delete_if { |k, v| k.is_a? String }.size,
types_stats
]
end
end
end
end
# encoders/term.rb
# By Rob Aldred (http://robaldred.co.uk)
# Based on idea by Nathan Weizenbaum (http://nex-3.com)
# MIT License (http://www.opensource.org/licenses/mit-license.php)
#
# A CodeRay encoder that outputs code highlighted for a color terminal.
# Check out http://robaldred.co.uk
module CodeRay
module Encoders
class Term < Encoder
register_for :term
TOKEN_COLORS = {
:annotation => '35',
:attribute_name => '33',
:attribute_name_fat => '33',
:attribute_value => '31',
:attribute_value_fat => '31',
:bin => '1;35',
:char => {:self => '36', :delimiter => '34'},
:class => '1;35',
:class_variable => '36',
:color => '32',
:comment => '37',
:complex => '34',
:constant => ['34', '4'],
:decoration => '35',
:definition => '1;32',
:directive => ['32', '4'],
:doc => '46',
:doctype => '1;30',
:doc_string => ['31', '4'],
:entity => '33',
:error => ['1;33', '41'],
:exception => '1;31',
:float => '1;35',
:function => '1;34',
:global_variable => '42',
:hex => '1;36',
:important => '1;31',
:include => '33',
:integer => '1;34',
:interpreted => '1;35',
:key => '35',
:label => '1;4',
:local_variable => '33',
:oct => '1;35',
:operator_name => '1;29',
:pre_constant => '1;36',
:pre_type => '1;30',
:predefined => ['4', '1;34'],
:preprocessor => '36',
:pseudo_class => '34',
:regexp => {
:content => '31',
:delimiter => '1;29',
:modifier => '35',
:function => '1;29'
},
:reserved => '1;31',
:shell => {
:self => '42',
:content => '1;29',
:delimiter => '37',
},
:string => {
:self => '32',
:modifier => '1;32',
:escape => '1;36',
:delimiter => '1;32',
},
:symbol => '1;32',
:tag => '34',
:tag_fat => '1;34',
:tag_special => ['34', '4'],
:type => '1;34',
:value => '36',
:variable => '34',
:insert => '42',
:delete => '41',
:change => '44',
:head => '45',
}
TOKEN_COLORS[:keyword] = TOKEN_COLORS[:reserved]
TOKEN_COLORS[:method] = TOKEN_COLORS[:function]
TOKEN_COLORS[:imaginary] = TOKEN_COLORS[:complex]
TOKEN_COLORS[:open] = TOKEN_COLORS[:close] = TOKEN_COLORS[:nesting_delimiter] = TOKEN_COLORS[:escape] = TOKEN_COLORS[:delimiter]
protected
def setup(options)
@out = ''
@opened = [nil]
@subcolors = nil
end
def finish(options)
super
end
def token text, type = :plain
case text
when nil
# raise 'Token with nil as text was given: %p' % [[text, type]]
when String
if color = (@subcolors || TOKEN_COLORS)[type]
color = color[:self] || return if Hash === color
@out << col(color) + text.gsub("\n", col(0) + "\n" + col(color)) + col(0)
@out << col(@subcolors[:self]) if @subcolors && @subcolors[:self]
else
@out << text
end
# token groups, eg. strings
when :open
@opened[0] = type
if color = TOKEN_COLORS[type]
if Hash === color
@subcolors = color
@out << col(color[:self]) if color[:self]
else
@subcolors = {}
@out << col(color)
end
end
@opened << type
when :close
if @opened.empty?
# nothing to close
else
@out << col(0) if (@subcolors || {})[:self]
@subcolors = nil
@opened.pop
end
# whole lines to be highlighted, eg. a added/modified/deleted lines in a diff
when :begin_line
when :end_line
else
raise 'unknown token kind: %p' % [text]
end
end
private
def col(color)
Array(color).map { |c| "\e[#{c}m" }.join
end
end
end
end
\ No newline at end of file
module CodeRay
module Encoders
class Text < Encoder
include Streamable
register_for :text
FILE_EXTENSION = 'txt'
DEFAULT_OPTIONS = {
:separator => ''
}
protected
def setup options
super
@sep = options[:separator]
end
def text_token text, kind
text + @sep
end
def finish options
super.chomp @sep
end
end
end
end
($:.unshift '../..'; require 'coderay') unless defined? CodeRay
module CodeRay
module Encoders
load :filter
class TokenClassFilter < Filter
include Streamable
register_for :token_class_filter
DEFAULT_OPTIONS = {
:exclude => [],
:include => :all
}
protected
def setup options
super
@exclude = options[:exclude]
@exclude = Array(@exclude) unless @exclude == :all
@include = options[:include]
@include = Array(@include) unless @include == :all
end
def include_text_token? text, kind
(@include == :all || @include.include?(kind)) &&
!(@exclude == :all || @exclude.include?(kind))
end
end
end
end
if $0 == __FILE__
$VERBOSE = true
$: << File.join(File.dirname(__FILE__), '..')
eval DATA.read, nil, $0, __LINE__ + 4
end
__END__
require 'test/unit'
class TokenClassFilterTest < Test::Unit::TestCase
def test_creation
assert CodeRay::Encoders::TokenClassFilter < CodeRay::Encoders::Encoder
assert CodeRay::Encoders::TokenClassFilter < CodeRay::Encoders::Filter
filter = nil
assert_nothing_raised do
filter = CodeRay.encoder :token_class_filter
end
assert_instance_of CodeRay::Encoders::TokenClassFilter, filter
end
def test_filtering_text_tokens
tokens = CodeRay::Tokens.new
for i in 1..10
tokens << [i.to_s, :index]
tokens << [' ', :space] if i < 10
end
assert_equal 10, CodeRay::Encoders::TokenClassFilter.new.encode_tokens(tokens, :exclude => :space).size
assert_equal 10, tokens.token_class_filter(:exclude => :space).size
assert_equal 9, CodeRay::Encoders::TokenClassFilter.new.encode_tokens(tokens, :include => :space).size
assert_equal 9, tokens.token_class_filter(:include => :space).size
assert_equal 0, CodeRay::Encoders::TokenClassFilter.new.encode_tokens(tokens, :exclude => :all).size
assert_equal 0, tokens.token_class_filter(:exclude => :all).size
end
def test_filtering_block_tokens
tokens = CodeRay::Tokens.new
10.times do |i|
tokens << [:open, :index]
tokens << [i.to_s, :content]
tokens << [:close, :index]
end
assert_equal 20, CodeRay::Encoders::TokenClassFilter.new.encode_tokens(tokens, :include => :blubb).size
assert_equal 20, tokens.token_class_filter(:include => :blubb).size
assert_equal 30, CodeRay::Encoders::TokenClassFilter.new.encode_tokens(tokens, :exclude => :index).size
assert_equal 30, tokens.token_class_filter(:exclude => :index).size
end
end
module CodeRay
module Encoders
# = XML Encoder
#
# Uses REXML. Very slow.
class XML < Encoder
include Streamable
register_for :xml
FILE_EXTENSION = 'xml'
require 'rexml/document'
DEFAULT_OPTIONS = {
:tab_width => 8,
:pretty => -1,
:transitive => false,
}
protected
def setup options
@doc = REXML::Document.new
@doc << REXML::XMLDecl.new
@tab_width = options[:tab_width]
@root = @node = @doc.add_element('coderay-tokens')
end
def finish options
@out = ''
@doc.write @out, options[:pretty], options[:transitive], true
@out
end
def text_token text, kind
if kind == :space
token = @node
else
token = @node.add_element kind.to_s
end
text.scan(/(\x20+)|(\t+)|(\n)|[^\x20\t\n]+/) do |space, tab, nl|
case
when space
token << REXML::Text.new(space, true)
when tab
token << REXML::Text.new(tab, true)
when nl
token << REXML::Text.new(nl, true)
else
token << REXML::Text.new($&)
end
end
end
def open_token kind
@node = @node.add_element kind.to_s
end
def close_token kind
if @node == @root
raise 'no token to close!'
end
@node = @node.parent
end
end
end
end
module CodeRay
module Encoders
# = YAML Encoder
#
# Slow.
class YAML < Encoder
register_for :yaml
FILE_EXTENSION = 'yaml'
protected
def compile tokens, options
require 'yaml'
@out = tokens.to_a.to_yaml
end
end
end
end
module CodeRay
# A little hack to enable CodeRay highlighting in RedCloth.
#
# Usage:
# require 'coderay'
# require 'coderay/for_redcloth'
# RedCloth.new('@[ruby]puts "Hello, World!"@').to_html
#
# Make sure you have RedCloth 4.0.3 activated, for example by calling
# require 'rubygems'
# before RedCloth is loaded and before calling CodeRay.for_redcloth.
module ForRedCloth
def self.install
gem 'RedCloth', '>= 4.0.3' if defined? gem
require 'redcloth'
unless RedCloth::VERSION.to_s >= '4.0.3'
if defined? gem
raise 'CodeRay.for_redcloth needs RedCloth version 4.0.3 or later. ' +
"You have #{RedCloth::VERSION}. Please gem install RedCloth."
else
$".delete 'redcloth.rb' # sorry, but it works
require 'rubygems'
return install # retry
end
end
unless RedCloth::VERSION.to_s >= '4.2.2'
warn 'CodeRay.for_redcloth works best with RedCloth version 4.2.2 or later.'
end
RedCloth::TextileDoc.send :include, ForRedCloth::TextileDoc
RedCloth::Formatters::HTML.module_eval do
def unescape(html)
replacements = {
'&amp;' => '&',
'&quot;' => '"',
'&gt;' => '>',
'&lt;' => '<',
}
html.gsub(/&(?:amp|quot|[gl]t);/) { |entity| replacements[entity] }
end
undef code, bc_open, bc_close, escape_pre
def code(opts) # :nodoc:
opts[:block] = true
if !opts[:lang] && RedCloth::VERSION.to_s >= '4.2.0'
# simulating pre-4.2 behavior
if opts[:text].sub!(/\A\[(\w+)\]/, '')
if CodeRay::Scanners[$1].plugin_id == 'plaintext'
opts[:text] = $& + opts[:text]
else
opts[:lang] = $1
end
end
end
if opts[:lang] && !filter_coderay
require 'coderay'
@in_bc ||= nil
format = @in_bc ? :div : :span
opts[:text] = unescape(opts[:text]) unless @in_bc
highlighted_code = CodeRay.encode opts[:text], opts[:lang], format, :stream => true
highlighted_code.sub!(/\A<(span|div)/) { |m| m + pba(@in_bc || opts) }
highlighted_code
else
"<code#{pba(opts)}>#{opts[:text]}</code>"
end
end
def bc_open(opts) # :nodoc:
opts[:block] = true
@in_bc = opts
opts[:lang] ? '' : "<pre#{pba(opts)}>"
end
def bc_close(opts) # :nodoc:
opts = @in_bc
@in_bc = nil
opts[:lang] ? '' : "</pre>\n"
end
def escape_pre(text)
if @in_bc ||= nil
text
else
html_esc(text, :html_escape_preformatted)
end
end
end
end
module TextileDoc # :nodoc:
attr_accessor :filter_coderay
end
end
end
CodeRay::ForRedCloth.install
\ No newline at end of file
#!/usr/bin/env ruby
module CodeRay
# = FileType
#
# A simple filetype recognizer.
#
# Copyright (c) 2006 by murphy (Kornelius Kalnbach) <murphy rubychan de>
#
# License:: LGPL / ask the author
# Version:: 0.1 (2005-09-01)
#
# == Documentation
#
# # determine the type of the given
# lang = FileType[ARGV.first]
#
# # return :plaintext if the file type is unknown
# lang = FileType.fetch ARGV.first, :plaintext
#
# # try the shebang line, too
# lang = FileType.fetch ARGV.first, :plaintext, true
module FileType
UnknownFileType = Class.new Exception
class << self
# Try to determine the file type of the file.
#
# +filename+ is a relative or absolute path to a file.
#
# The file itself is only accessed when +read_shebang+ is set to true.
# That means you can get filetypes from files that don't exist.
def [] filename, read_shebang = false
name = File.basename filename
ext = File.extname(name).sub(/^\./, '') # from last dot, delete the leading dot
ext2 = filename.to_s[/\.(.*)/, 1] # from first dot
type =
TypeFromExt[ext] ||
TypeFromExt[ext.downcase] ||
(TypeFromExt[ext2] if ext2) ||
(TypeFromExt[ext2.downcase] if ext2) ||
TypeFromName[name] ||
TypeFromName[name.downcase]
type ||= shebang(filename) if read_shebang
type
end
def shebang filename
begin
File.open filename, 'r' do |f|
if first_line = f.gets
if type = first_line[TypeFromShebang]
type.to_sym
end
end
end
rescue IOError
nil
end
end
# This works like Hash#fetch.
#
# If the filetype cannot be found, the +default+ value
# is returned.
def fetch filename, default = nil, read_shebang = false
if default and block_given?
warn 'block supersedes default value argument'
end
unless type = self[filename, read_shebang]
return yield if block_given?
return default if default
raise UnknownFileType, 'Could not determine type of %p.' % filename
end
type
end
end
TypeFromExt = {
'c' => :c,
'css' => :css,
'diff' => :diff,
'dpr' => :delphi,
'groovy' => :groovy,
'gvy' => :groovy,
'h' => :c,
'htm' => :html,
'html' => :html,
'html.erb' => :rhtml,
'java' => :java,
'js' => :java_script,
'json' => :json,
'mab' => :ruby,
'pas' => :delphi,
'patch' => :diff,
'php' => :php,
'php3' => :php,
'php4' => :php,
'php5' => :php,
'py' => :python,
'py3' => :python,
'pyw' => :python,
'rake' => :ruby,
'raydebug' => :debug,
'rb' => :ruby,
'rbw' => :ruby,
'rhtml' => :rhtml,
'rxml' => :ruby,
'sch' => :scheme,
'sql' => :sql,
'ss' => :scheme,
'xhtml' => :xhtml,
'xml' => :xml,
'yaml' => :yaml,
'yml' => :yaml,
}
for cpp_alias in %w[cc cpp cp cxx c++ C hh hpp h++ cu]
TypeFromExt[cpp_alias] = :cpp
end
TypeFromShebang = /\b(?:ruby|perl|python|sh)\b/
TypeFromName = {
'Rakefile' => :ruby,
'Rantfile' => :ruby,
}
end
end
if $0 == __FILE__
$VERBOSE = true
eval DATA.read, nil, $0, __LINE__ + 4
end
__END__
require 'test/unit'
class FileTypeTests < Test::Unit::TestCase
include CodeRay
def test_fetch
assert_raise FileType::UnknownFileType do
FileType.fetch ''
end
assert_throws :not_found do
FileType.fetch '.' do
throw :not_found
end
end
assert_equal :default, FileType.fetch('c', :default)
stderr, fake_stderr = $stderr, Object.new
$err = ''
def fake_stderr.write x
$err << x
end
$stderr = fake_stderr
FileType.fetch('c', :default) { }
assert_equal "block supersedes default value argument\n", $err
$stderr = stderr
end
def test_ruby
assert_equal :ruby, FileType['test.rb']
assert_equal :ruby, FileType['test.java.rb']
assert_equal :java, FileType['test.rb.java']
assert_equal :ruby, FileType['C:\\Program Files\\x\\y\\c\\test.rbw']
assert_equal :ruby, FileType['/usr/bin/something/Rakefile']
assert_equal :ruby, FileType['~/myapp/gem/Rantfile']
assert_equal :ruby, FileType['./lib/tasks\repository.rake']
assert_not_equal :ruby, FileType['test_rb']
assert_not_equal :ruby, FileType['Makefile']
assert_not_equal :ruby, FileType['set.rb/set']
assert_not_equal :ruby, FileType['~/projects/blabla/rb']
end
def test_c
assert_equal :c, FileType['test.c']
assert_equal :c, FileType['C:\\Program Files\\x\\y\\c\\test.h']
assert_not_equal :c, FileType['test_c']
assert_not_equal :c, FileType['Makefile']
assert_not_equal :c, FileType['set.h/set']
assert_not_equal :c, FileType['~/projects/blabla/c']
end
def test_cpp
assert_equal :cpp, FileType['test.c++']
assert_equal :cpp, FileType['test.cxx']
assert_equal :cpp, FileType['test.hh']
assert_equal :cpp, FileType['test.hpp']
assert_equal :cpp, FileType['test.cu']
assert_equal :cpp, FileType['test.C']
assert_not_equal :cpp, FileType['test.c']
assert_not_equal :cpp, FileType['test.h']
end
def test_html
assert_equal :html, FileType['test.htm']
assert_equal :xhtml, FileType['test.xhtml']
assert_equal :xhtml, FileType['test.html.xhtml']
assert_equal :rhtml, FileType['_form.rhtml']
assert_equal :rhtml, FileType['_form.html.erb']
end
def test_yaml
assert_equal :yaml, FileType['test.yml']
assert_equal :yaml, FileType['test.yaml']
assert_equal :yaml, FileType['my.html.yaml']
assert_not_equal :yaml, FileType['YAML']
end
def test_pathname
require 'pathname'
pn = Pathname.new 'test.rb'
assert_equal :ruby, FileType[pn]
dir = Pathname.new '/etc/var/blubb'
assert_equal :ruby, FileType[dir + pn]
assert_equal :cpp, FileType[dir + 'test.cpp']
end
def test_no_shebang
dir = './test'
if File.directory? dir
Dir.chdir dir do
assert_equal :c, FileType['test.c']
end
end
end
def test_shebang_empty_file
require 'tmpdir'
tmpfile = File.join(Dir.tmpdir, 'bla')
File.open(tmpfile, 'w') { } # touch
assert_equal nil, FileType[tmpfile]
end
def test_shebang
require 'tmpdir'
tmpfile = File.join(Dir.tmpdir, 'bla')
File.open(tmpfile, 'w') { |f| f.puts '#!/usr/bin/env ruby' }
assert_equal :ruby, FileType[tmpfile, true]
end
end
# =GZip Simple
#
# A simplified interface to the gzip library +zlib+ (from the Ruby Standard Library.)
#
# Author: murphy (mail to murphy rubychan de)
#
# Version: 0.2 (2005.may.28)
#
# ==Documentation
#
# See +GZip+ module and the +String+ extensions.
#
module GZip
require 'zlib'
# The default zipping level. 7 zips good and fast.
DEFAULT_GZIP_LEVEL = 7
# Unzips the given string +s+.
#
# Example:
# require 'gzip_simple'
# print GZip.gunzip(File.read('adresses.gz'))
def GZip.gunzip s
Zlib::Inflate.inflate s
end
# Zips the given string +s+.
#
# Example:
# require 'gzip_simple'
# File.open('adresses.gz', 'w') do |file
# file.write GZip.gzip('Mum: 0123 456 789', 9)
# end
#
# If you provide a +level+, you can control how strong
# the string is compressed:
# - 0: no compression, only convert to gzip format
# - 1: compress fast
# - 7: compress more, but still fast (default)
# - 8: compress more, slower
# - 9: compress best, very slow
def GZip.gzip s, level = DEFAULT_GZIP_LEVEL
Zlib::Deflate.new(level).deflate s, Zlib::FINISH
end
end
# String extensions to use the GZip module.
#
# The methods gzip and gunzip provide an even more simple
# interface to the ZLib:
#
# # create a big string
# x = 'a' * 1000
#
# # zip it
# x_gz = x.gzip
#
# # test the result
# puts 'Zipped %d bytes to %d bytes.' % [x.size, x_gz.size]
# #-> Zipped 1000 bytes to 19 bytes.
#
# # unzipping works
# p x_gz.gunzip == x #-> true
class String
# Returns the string, unzipped.
# See GZip.gunzip
def gunzip
GZip.gunzip self
end
# Replaces the string with its unzipped value.
# See GZip.gunzip
def gunzip!
replace gunzip
end
# Returns the string, zipped.
# +level+ is the gzip compression level, see GZip.gzip.
def gzip level = GZip::DEFAULT_GZIP_LEVEL
GZip.gzip self, level
end
# Replaces the string with its zipped value.
# See GZip.gzip.
def gzip!(*args)
replace gzip(*args)
end
end
if $0 == __FILE__
eval DATA.read, nil, $0, __LINE__+4
end
__END__
#CODE
# Testing / Benchmark
x = 'a' * 1000
x_gz = x.gzip
puts 'Zipped %d bytes to %d bytes.' % [x.size, x_gz.size] #-> Zipped 1000 bytes to 19 bytes.
p x_gz.gunzip == x #-> true
require 'benchmark'
INFO = 'packed to %0.3f%%' # :nodoc:
x = Array.new(100000) { rand(255).chr + 'aaaaaaaaa' + rand(255).chr }.join
Benchmark.bm(10) do |bm|
for level in 0..9
bm.report "zip #{level}" do
$x = x.gzip level
end
puts INFO % [100.0 * $x.size / x.size]
end
bm.report 'zip' do
$x = x.gzip
end
puts INFO % [100.0 * $x.size / x.size]
bm.report 'unzip' do
$x.gunzip
end
end
module CodeRay
# = PluginHost
#
# A simple subclass plugin system.
#
# Example:
# class Generators < PluginHost
# plugin_path 'app/generators'
# end
#
# class Generator
# extend Plugin
# PLUGIN_HOST = Generators
# end
#
# class FancyGenerator < Generator
# register_for :fancy
# end
#
# Generators[:fancy] #-> FancyGenerator
# # or
# CodeRay.require_plugin 'Generators/fancy'
module PluginHost
# Raised if Encoders::[] fails because:
# * a file could not be found
# * the requested Encoder is not registered
PluginNotFound = Class.new Exception
HostNotFound = Class.new Exception
PLUGIN_HOSTS = []
PLUGIN_HOSTS_BY_ID = {} # dummy hash
# Loads all plugins using list and load.
def load_all
for plugin in list
load plugin
end
end
# Returns the Plugin for +id+.
#
# Example:
# yaml_plugin = MyPluginHost[:yaml]
def [] id, *args, &blk
plugin = validate_id(id)
begin
plugin = plugin_hash.[] plugin, *args, &blk
end while plugin.is_a? Symbol
plugin
end
# Alias for +[]+.
alias load []
def require_helper plugin_id, helper_name
path = path_to File.join(plugin_id, helper_name)
require path
end
class << self
# Adds the module/class to the PLUGIN_HOSTS list.
def extended mod
PLUGIN_HOSTS << mod
end
# Warns you that you should not #include this module.
def included mod
warn "#{name} should not be included. Use extend."
end
# Find the PluginHost for host_id.
def host_by_id host_id
unless PLUGIN_HOSTS_BY_ID.default_proc
ph = Hash.new do |h, a_host_id|
for host in PLUGIN_HOSTS
h[host.host_id] = host
end
h.fetch a_host_id, nil
end
PLUGIN_HOSTS_BY_ID.replace ph
end
PLUGIN_HOSTS_BY_ID[host_id]
end
end
# The path where the plugins can be found.
def plugin_path *args
unless args.empty?
@plugin_path = File.expand_path File.join(*args)
load_map
end
@plugin_path
end
# The host's ID.
#
# If PLUGIN_HOST_ID is not set, it is simply the class name.
def host_id
if self.const_defined? :PLUGIN_HOST_ID
self::PLUGIN_HOST_ID
else
name
end
end
# Map a plugin_id to another.
#
# Usage: Put this in a file plugin_path/_map.rb.
#
# class MyColorHost < PluginHost
# map :navy => :dark_blue,
# :maroon => :brown,
# :luna => :moon
# end
def map hash
for from, to in hash
from = validate_id from
to = validate_id to
plugin_hash[from] = to unless plugin_hash.has_key? from
end
end
# Define the default plugin to use when no plugin is found
# for a given id.
#
# See also map.
#
# class MyColorHost < PluginHost
# map :navy => :dark_blue
# default :gray
# end
def default id = nil
if id
id = validate_id id
plugin_hash[nil] = id
else
plugin_hash[nil]
end
end
# Every plugin must register itself for one or more
# +ids+ by calling register_for, which calls this method.
#
# See Plugin#register_for.
def register plugin, *ids
for id in ids
unless id.is_a? Symbol
raise ArgumentError,
"id must be a Symbol, but it was a #{id.class}"
end
plugin_hash[validate_id(id)] = plugin
end
end
# A Hash of plugion_id => Plugin pairs.
def plugin_hash
@plugin_hash ||= create_plugin_hash
end
# Returns an array of all .rb files in the plugin path.
#
# The extension .rb is not included.
def list
Dir[path_to('*')].select do |file|
File.basename(file)[/^(?!_)\w+\.rb$/]
end.map do |file|
File.basename file, '.rb'
end
end
# Makes a map of all loaded plugins.
def inspect
map = plugin_hash.dup
map.each do |id, plugin|
map[id] = plugin.to_s[/(?>\w+)$/]
end
"#{name}[#{host_id}]#{map.inspect}"
end
protected
# Created a new plugin list and stores it to @plugin_hash.
def create_plugin_hash
@plugin_hash =
Hash.new do |h, plugin_id|
id = validate_id(plugin_id)
path = path_to id
begin
require path
rescue LoadError => boom
if h.has_key? nil # default plugin
h[id] = h[nil]
else
raise PluginNotFound, 'Could not load plugin %p: %s' % [id, boom]
end
else
# Plugin should have registered by now
unless h.has_key? id
raise PluginNotFound,
"No #{self.name} plugin for #{id.inspect} found in #{path}."
end
end
h[id]
end
end
# Loads the map file (see map).
#
# This is done automatically when plugin_path is called.
def load_map
mapfile = path_to '_map'
if File.exist? mapfile
require mapfile
elsif $VERBOSE
warn 'no _map.rb found for %s' % name
end
end
# Returns the Plugin for +id+.
# Use it like Hash#fetch.
#
# Example:
# yaml_plugin = MyPluginHost[:yaml, :default]
def fetch id, *args, &blk
plugin_hash.fetch validate_id(id), *args, &blk
end
# Returns the expected path to the plugin file for the given id.
def path_to plugin_id
File.join plugin_path, "#{plugin_id}.rb"
end
# Converts +id+ to a Symbol if it is a String,
# or returns +id+ if it already is a Symbol.
#
# Raises +ArgumentError+ for all other objects, or if the
# given String includes non-alphanumeric characters (\W).
def validate_id id
if id.is_a? Symbol or id.nil?
id
elsif id.is_a? String
if id[/\w+/] == id
id.downcase.to_sym
else
raise ArgumentError, "Invalid id: '#{id}' given."
end
else
raise ArgumentError,
"String or Symbol expected, but #{id.class} given."
end
end
end
# = Plugin
#
# Plugins have to include this module.
#
# IMPORTANT: use extend for this module.
#
# Example: see PluginHost.
module Plugin
def included mod
warn "#{name} should not be included. Use extend."
end
# Register this class for the given langs.
# Example:
# class MyPlugin < PluginHost::BaseClass
# register_for :my_id
# ...
# end
#
# See PluginHost.register.
def register_for *ids
plugin_host.register self, *ids
end
# Returns the title of the plugin, or sets it to the
# optional argument +title+.
def title title = nil
if title
@title = title.to_s
else
@title ||= name[/([^:]+)$/, 1]
end
end
# The host for this Plugin class.
def plugin_host host = nil
if host and not host.is_a? PluginHost
raise ArgumentError,
"PluginHost expected, but #{host.class} given."
end
self.const_set :PLUGIN_HOST, host if host
self::PLUGIN_HOST
end
# Require some helper files.
#
# Example:
#
# class MyPlugin < PluginHost::BaseClass
# register_for :my_id
# helper :my_helper
#
# The above example loads the file myplugin/my_helper.rb relative to the
# file in which MyPlugin was defined.
#
# You can also load a helper from a different plugin:
#
# helper 'other_plugin/helper_name'
def helper *helpers
for helper in helpers
if helper.is_a?(String) && helper[/\//]
self::PLUGIN_HOST.require_helper $`, $'
else
self::PLUGIN_HOST.require_helper plugin_id, helper.to_s
end
end
end
# Returns the pulgin id used by the engine.
def plugin_id
name[/\w+$/].downcase
end
end
# Convenience method for plugin loading.
# The syntax used is:
#
# CodeRay.require_plugin '<Host ID>/<Plugin ID>'
#
# Returns the loaded plugin.
def self.require_plugin path
host_id, plugin_id = path.split '/', 2
host = PluginHost.host_by_id(host_id)
raise PluginHost::HostNotFound,
"No host for #{host_id.inspect} found." unless host
host.load plugin_id
end
end
\ No newline at end of file
module CodeRay
# = WordList
#
# <b>A Hash subclass designed for mapping word lists to token types.</b>
#
# Copyright (c) 2006 by murphy (Kornelius Kalnbach) <murphy rubychan de>
#
# License:: LGPL / ask the author
# Version:: 1.1 (2006-Oct-19)
#
# A WordList is a Hash with some additional features.
# It is intended to be used for keyword recognition.
#
# WordList is highly optimized to be used in Scanners,
# typically to decide whether a given ident is a special token.
#
# For case insensitive words use CaseIgnoringWordList.
#
# Example:
#
# # define word arrays
# RESERVED_WORDS = %w[
# asm break case continue default do else
# ...
# ]
#
# PREDEFINED_TYPES = %w[
# int long short char void
# ...
# ]
#
# PREDEFINED_CONSTANTS = %w[
# EOF NULL ...
# ]
#
# # make a WordList
# IDENT_KIND = WordList.new(:ident).
# add(RESERVED_WORDS, :reserved).
# add(PREDEFINED_TYPES, :pre_type).
# add(PREDEFINED_CONSTANTS, :pre_constant)
#
# ...
#
# def scan_tokens tokens, options
# ...
#
# elsif scan(/[A-Za-z_][A-Za-z_0-9]*/)
# # use it
# kind = IDENT_KIND[match]
# ...
class WordList < Hash
# Creates a new WordList with +default+ as default value.
#
# You can activate +caching+ to store the results for every [] request.
#
# With caching, methods like +include?+ or +delete+ may no longer behave
# as you expect. Therefore, it is recommended to use the [] method only.
def initialize default = false, caching = false, &block
if block
raise ArgumentError, 'Can\'t combine block with caching.' if caching
super(&block)
else
if caching
super() do |h, k|
h[k] = h.fetch k, default
end
else
super default
end
end
end
# Add words to the list and associate them with +kind+.
#
# Returns +self+, so you can concat add calls.
def add words, kind = true
words.each do |word|
self[word] = kind
end
self
end
end
# A CaseIgnoringWordList is like a WordList, only that
# keys are compared case-insensitively.
#
# Ignoring the text case is realized by sending the +downcase+ message to
# all keys.
#
# Caching usually makes a CaseIgnoringWordList faster, but it has to be
# activated explicitely.
class CaseIgnoringWordList < WordList
# Creates a new case-insensitive WordList with +default+ as default value.
#
# You can activate caching to store the results for every [] request.
# This speeds up subsequent lookups for the same word, but also
# uses memory.
def initialize default = false, caching = false
if caching
super(default, false) do |h, k|
h[k] = h.fetch k.downcase, default
end
else
super(default, false)
extend Uncached
end
end
module Uncached # :nodoc:
def [] key
super(key.downcase)
end
end
# Add +words+ to the list and associate them with +kind+.
def add words, kind = true
words.each do |word|
self[word.downcase] = kind
end
self
end
end
end
__END__
# check memory consumption
END {
ObjectSpace.each_object(CodeRay::CaseIgnoringWordList) do |wl|
p wl.inject(0) { |memo, key, value| memo + key.size + 24 }
end
}
\ No newline at end of file
module CodeRay
require 'coderay/helpers/plugin'
# = Scanners
#
# This module holds the Scanner class and its subclasses.
# For example, the Ruby scanner is named CodeRay::Scanners::Ruby
# can be found in coderay/scanners/ruby.
#
# Scanner also provides methods and constants for the register
# mechanism and the [] method that returns the Scanner class
# belonging to the given lang.
#
# See PluginHost.
module Scanners
extend PluginHost
plugin_path File.dirname(__FILE__), 'scanners'
require 'strscan'
# = Scanner
#
# The base class for all Scanners.
#
# It is a subclass of Ruby's great +StringScanner+, which
# makes it easy to access the scanning methods inside.
#
# It is also +Enumerable+, so you can use it like an Array of
# Tokens:
#
# require 'coderay'
#
# c_scanner = CodeRay::Scanners[:c].new "if (*p == '{') nest++;"
#
# for text, kind in c_scanner
# puts text if kind == :operator
# end
#
# # prints: (*==)++;
#
# OK, this is a very simple example :)
# You can also use +map+, +any?+, +find+ and even +sort_by+,
# if you want.
class Scanner < StringScanner
extend Plugin
plugin_host Scanners
# Raised if a Scanner fails while scanning
ScanError = Class.new(Exception)
require 'coderay/helpers/word_list'
# The default options for all scanner classes.
#
# Define @default_options for subclasses.
DEFAULT_OPTIONS = { :stream => false }
KINDS_NOT_LOC = [:comment, :doctype]
class << self
# Returns if the Scanner can be used in streaming mode.
def streamable?
is_a? Streamable
end
def normify code
code = code.to_s
if code.respond_to?(:encoding) && (code.encoding.name != 'UTF-8' || !code.valid_encoding?)
code = code.dup
original_encoding = code.encoding
code.force_encoding 'Windows-1252'
unless code.valid_encoding?
code.force_encoding original_encoding
if code.encoding.name == 'UTF-8'
code.encode! 'UTF-16BE', :invalid => :replace, :undef => :replace, :replace => '?'
end
code.encode! 'UTF-8', :invalid => :replace, :undef => :replace, :replace => '?'
end
end
code.to_unix
end
def file_extension extension = nil
if extension
@file_extension = extension.to_s
else
@file_extension ||= plugin_id.to_s
end
end
end
=begin
## Excluded for speed reasons; protected seems to make methods slow.
# Save the StringScanner methods from being called.
# This would not be useful for highlighting.
strscan_public_methods =
StringScanner.instance_methods -
StringScanner.ancestors[1].instance_methods
protected(*strscan_public_methods)
=end
# Create a new Scanner.
#
# * +code+ is the input String and is handled by the superclass
# StringScanner.
# * +options+ is a Hash with Symbols as keys.
# It is merged with the default options of the class (you can
# overwrite default options here.)
# * +block+ is the callback for streamed highlighting.
#
# If you set :stream to +true+ in the options, the Scanner uses a
# TokenStream with the +block+ as callback to handle the tokens.
#
# Else, a Tokens object is used.
def initialize code='', options = {}, &block
raise "I am only the basic Scanner class. I can't scan "\
"anything. :( Use my subclasses." if self.class == Scanner
@options = self.class::DEFAULT_OPTIONS.merge options
super Scanner.normify(code)
@tokens = options[:tokens]
if @options[:stream]
warn "warning in CodeRay::Scanner.new: :stream is set, "\
"but no block was given" unless block_given?
raise NotStreamableError, self unless kind_of? Streamable
@tokens ||= TokenStream.new(&block)
else
warn "warning in CodeRay::Scanner.new: Block given, "\
"but :stream is #{@options[:stream]}" if block_given?
@tokens ||= Tokens.new
end
@tokens.scanner = self
setup
end
def reset
super
reset_instance
end
def string= code
code = Scanner.normify(code)
if defined?(RUBY_DESCRIPTION) && RUBY_DESCRIPTION['rubinius 1.0.1']
reset_state
@string = code
else
super code
end
reset_instance
end
# More mnemonic accessor name for the input string.
alias code string
alias code= string=
# Returns the Plugin ID for this scanner.
def lang
self.class.plugin_id
end
# Scans the code and returns all tokens in a Tokens object.
def tokenize new_string=nil, options = {}
options = @options.merge(options)
self.string = new_string if new_string
@cached_tokens =
if @options[:stream] # :stream must have been set already
reset unless new_string
scan_tokens @tokens, options
@tokens
else
scan_tokens @tokens, options
end
end
def tokens
@cached_tokens ||= tokenize
end
# Whether the scanner is in streaming mode.
def streaming?
!!@options[:stream]
end
# Traverses the tokens.
def each &block
raise ArgumentError,
'Cannot traverse TokenStream.' if @options[:stream]
tokens.each(&block)
end
include Enumerable
# The current line position of the scanner.
#
# Beware, this is implemented inefficiently. It should be used
# for debugging only.
def line
string[0..pos].count("\n") + 1
end
def column pos = self.pos
return 0 if pos <= 0
string = string()
if string.respond_to?(:bytesize) && (defined?(@bin_string) || string.bytesize != string.size)
@bin_string ||= string.dup.force_encoding('binary')
string = @bin_string
end
pos - (string.rindex(?\n, pos) || 0)
end
def marshal_dump
@options
end
def marshal_load options
@options = options
end
protected
# Can be implemented by subclasses to do some initialization
# that has to be done once per instance.
#
# Use reset for initialization that has to be done once per
# scan.
def setup
end
# This is the central method, and commonly the only one a
# subclass implements.
#
# Subclasses must implement this method; it must return +tokens+
# and must only use Tokens#<< for storing scanned tokens!
def scan_tokens tokens, options
raise NotImplementedError,
"#{self.class}#scan_tokens not implemented."
end
def reset_instance
@tokens.clear unless @options[:keep_tokens]
@cached_tokens = nil
@bin_string = nil if defined? @bin_string
end
# Scanner error with additional status information
def raise_inspect msg, tokens, state = 'No state given!', ambit = 30
raise ScanError, <<-EOE % [
***ERROR in %s: %s (after %d tokens)
tokens:
%s
current line: %d column: %d pos: %d
matched: %p state: %p
bol? = %p, eos? = %p
surrounding code:
%p ~~ %p
***ERROR***
EOE
File.basename(caller[0]),
msg,
tokens.size,
tokens.last(10).map { |t| t.inspect }.join("\n"),
line, column, pos,
matched, state, bol?, eos?,
string[pos - ambit, ambit],
string[pos, ambit],
]
end
end
end
end
class String
# I love this hack. It seems to silence all dos/unix/mac newline problems.
def to_unix
if index ?\r
gsub(/\r\n?/, "\n")
else
self
end
end
end
module CodeRay
module Scanners
map \
:h => :c,
:cplusplus => :cpp,
:'c++' => :cpp,
:ecma => :java_script,
:ecmascript => :java_script,
:ecma_script => :java_script,
:irb => :ruby,
:javascript => :java_script,
:js => :java_script,
:nitro => :nitro_xhtml,
:pascal => :delphi,
:plain => :plaintext,
:xhtml => :html,
:yml => :yaml
default :plain
end
end
module CodeRay
module Scanners
class C < Scanner
include Streamable
register_for :c
file_extension 'c'
RESERVED_WORDS = [
'asm', 'break', 'case', 'continue', 'default', 'do',
'else', 'enum', 'for', 'goto', 'if', 'return',
'sizeof', 'struct', 'switch', 'typedef', 'union', 'while',
'restrict', # added in C99
]
PREDEFINED_TYPES = [
'int', 'long', 'short', 'char',
'signed', 'unsigned', 'float', 'double',
'bool', 'complex', # added in C99
]
PREDEFINED_CONSTANTS = [
'EOF', 'NULL',
'true', 'false', # added in C99
]
DIRECTIVES = [
'auto', 'extern', 'register', 'static', 'void',
'const', 'volatile', # added in C89
'inline', # added in C99
]
IDENT_KIND = WordList.new(:ident).
add(RESERVED_WORDS, :reserved).
add(PREDEFINED_TYPES, :pre_type).
add(DIRECTIVES, :directive).
add(PREDEFINED_CONSTANTS, :pre_constant)
ESCAPE = / [rbfntv\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} /x
UNICODE_ESCAPE = / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} /x
def scan_tokens tokens, options
state = :initial
label_expected = true
case_expected = false
label_expected_before_preproc_line = nil
in_preproc_line = false
until eos?
kind = nil
match = nil
case state
when :initial
if match = scan(/ \s+ | \\\n /x)
if in_preproc_line && match != "\\\n" && match.index(?\n)
in_preproc_line = false
label_expected = label_expected_before_preproc_line
end
tokens << [match, :space]
next
elsif scan(%r! // [^\n\\]* (?: \\. [^\n\\]* )* | /\* (?: .*? \*/ | .* ) !mx)
kind = :comment
elsif match = scan(/ \# \s* if \s* 0 /x)
match << scan_until(/ ^\# (?:elif|else|endif) .*? $ | \z /xm) unless eos?
kind = :comment
elsif match = scan(/ [-+*=<>?:;,!&^|()\[\]{}~%]+ | \/=? | \.(?!\d) /x)
label_expected = match =~ /[;\{\}]/
if case_expected
label_expected = true if match == ':'
case_expected = false
end
kind = :operator
elsif match = scan(/ [A-Za-z_][A-Za-z_0-9]* /x)
kind = IDENT_KIND[match]
if kind == :ident && label_expected && !in_preproc_line && scan(/:(?!:)/)
kind = :label
match << matched
else
label_expected = false
if kind == :reserved
case match
when 'case', 'default'
case_expected = true
end
end
end
elsif scan(/\$/)
kind = :ident
elsif match = scan(/L?"/)
tokens << [:open, :string]
if match[0] == ?L
tokens << ['L', :modifier]
match = '"'
end
state = :string
kind = :delimiter
elsif scan(/#[ \t]*(\w*)/)
kind = :preprocessor
in_preproc_line = true
label_expected_before_preproc_line = label_expected
state = :include_expected if self[1] == 'include'
elsif scan(/ L?' (?: [^\'\n\\] | \\ #{ESCAPE} )? '? /ox)
label_expected = false
kind = :char
elsif scan(/0[xX][0-9A-Fa-f]+/)
label_expected = false
kind = :hex
elsif scan(/(?:0[0-7]+)(?![89.eEfF])/)
label_expected = false
kind = :oct
elsif scan(/(?:\d+)(?![.eEfF])L?L?/)
label_expected = false
kind = :integer
elsif scan(/\d[fF]?|\d*\.\d+(?:[eE][+-]?\d+)?[fF]?|\d+[eE][+-]?\d+[fF]?/)
label_expected = false
kind = :float
else
getch
kind = :error
end
when :string
if scan(/[^\\\n"]+/)
kind = :content
elsif scan(/"/)
tokens << ['"', :delimiter]
tokens << [:close, :string]
state = :initial
label_expected = false
next
elsif scan(/ \\ (?: #{ESCAPE} | #{UNICODE_ESCAPE} ) /mox)
kind = :char
elsif scan(/ \\ | $ /x)
tokens << [:close, :string]
kind = :error
state = :initial
label_expected = false
else
raise_inspect "else case \" reached; %p not handled." % peek(1), tokens
end
when :include_expected
if scan(/<[^>\n]+>?|"[^"\n\\]*(?:\\.[^"\n\\]*)*"?/)
kind = :include
state = :initial
elsif match = scan(/\s+/)
kind = :space
state = :initial if match.index ?\n
else
state = :initial
next
end
else
raise_inspect 'Unknown state', tokens
end
match ||= matched
if $CODERAY_DEBUG and not kind
raise_inspect 'Error token %p in line %d' %
[[match, kind], line], tokens
end
raise_inspect 'Empty token', tokens unless match
tokens << [match, kind]
end
if state == :string
tokens << [:close, :string]
end
tokens
end
end
end
end
This diff is collapsed.
This diff is collapsed.
module CodeRay
module Scanners
# = Debug Scanner
class Debug < Scanner
include Streamable
register_for :debug
file_extension 'raydebug'
title 'CodeRay Token Dump'
protected
def scan_tokens tokens, options
opened_tokens = []
until eos?
kind = nil
match = nil
if scan(/\s+/)
tokens << [matched, :space]
next
elsif scan(/ (\w+) \( ( [^\)\\]* ( \\. [^\)\\]* )* ) \) /x)
kind = self[1].to_sym
match = self[2].gsub(/\\(.)/, '\1')
elsif scan(/ (\w+) < /x)
kind = self[1].to_sym
opened_tokens << kind
match = :open
elsif !opened_tokens.empty? && scan(/ > /x)
kind = opened_tokens.pop || :error
match = :close
else
kind = :error
getch
end
match ||= matched
if $CODERAY_DEBUG and not kind
raise_inspect 'Error token %p in line %d' %
[[match, kind], line], tokens
end
raise_inspect 'Empty token', tokens unless match
tokens << [match, kind]
end
tokens
end
end
end
end
module CodeRay
module Scanners
class Diff < Scanner
register_for :diff
title 'diff output'
def scan_tokens tokens, options
line_kind = nil
state = :initial
until eos?
kind = match = nil
if match = scan(/\n/)
if line_kind
tokens << [:end_line, line_kind]
line_kind = nil
end
tokens << [match, :space]
next
end
case state
when :initial
if match = scan(/--- |\+\+\+ |=+|_+/)
tokens << [:begin_line, line_kind = :head]
tokens << [match, :head]
next unless match = scan(/.+/)
kind = :plain
elsif match = scan(/Index: |Property changes on: /)
tokens << [:begin_line, line_kind = :head]
tokens << [match, :head]
next unless match = scan(/.+/)
kind = :plain
elsif match = scan(/Added: /)
tokens << [:begin_line, line_kind = :head]
tokens << [match, :head]
next unless match = scan(/.+/)
kind = :plain
state = :added
elsif match = scan(/\\ /)
tokens << [:begin_line, line_kind = :change]
tokens << [match, :change]
next unless match = scan(/.+/)
kind = :plain
elsif match = scan(/@@(?>[^@\n]*)@@/)
if check(/\n|$/)
tokens << [:begin_line, line_kind = :change]
else
tokens << [:open, :change]
end
tokens << [match[0,2], :change]
tokens << [match[2...-2], :plain]
tokens << [match[-2,2], :change]
tokens << [:close, :change] unless line_kind
next unless match = scan(/.+/)
kind = :plain
elsif match = scan(/\+/)
tokens << [:begin_line, line_kind = :insert]
tokens << [match, :insert]
next unless match = scan(/.+/)
kind = :plain
elsif match = scan(/-/)
tokens << [:begin_line, line_kind = :delete]
tokens << [match, :delete]
next unless match = scan(/.+/)
kind = :plain
elsif scan(/ .*/)
kind = :comment
elsif scan(/.+/)
tokens << [:begin_line, line_kind = :comment]
kind = :plain
else
raise_inspect 'else case rached'
end
when :added
if match = scan(/ \+/)
tokens << [:begin_line, line_kind = :insert]
tokens << [match, :insert]
next unless match = scan(/.+/)
kind = :plain
else
state = :initial
next
end
end
match ||= matched
if $CODERAY_DEBUG and not kind
raise_inspect 'Error token %p in line %d' %
[[match, kind], line], tokens
end
raise_inspect 'Empty token', tokens unless match
tokens << [match, kind]
end
tokens << [:end_line, line_kind] if line_kind
tokens
end
end
end
end
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment