From a7d6b41d5bc6f075eba94bbbad363634894586f5 Mon Sep 17 00:00:00 2001 From: Eric Hodel Date: Mon, 4 Feb 2013 18:50:20 -0800 Subject: [PATCH 01/20] Fix CVE-2013-0256, an XSS exploit in darkfish.js This XSS exploit could lead to cookie disclosure on websites hosting generated RDoc content. This commit includes a patch rdoc users may apply to their generated content in CVE-2013-0256.rdoc. --- CVE-2013-0256.rdoc | 48 +++++++++++++++++++ History.rdoc | 7 +++ Manifest.txt | 1 + Rakefile | 1 + .../template/darkfish/js/darkfish.js | 16 ++++--- 5 files changed, 66 insertions(+), 7 deletions(-) create mode 100644 CVE-2013-0256.rdoc diff --git a/CVE-2013-0256.rdoc b/CVE-2013-0256.rdoc new file mode 100644 index 0000000000..79acbe86ac --- /dev/null +++ b/CVE-2013-0256.rdoc @@ -0,0 +1,48 @@ += RDoc 2.3.0 through 3.12 XSS Exploit + +RDoc documentation generated by rdoc 2.3.0 through rdoc 3.12 and prereleases up +to rdoc 4.0.0.preview2.1 are vulnerable to an XSS exploit. This exploit may +lead to cookie disclosure to third parties. + +The exploit exists in darkfish.js which is copied from the RDoc install +location to the generated documentation. + +RDoc is a static documentation generation tool. Patching the library itself +is insufficient to correct this exploit. Those hosting rdoc documentation will +need to apply the following patch. If applied while ignoring whitespace, this +patch will correct all affected versions: + + diff --git darkfish.js darkfish.js + index 4be722f..f26fd45 100644 + --- darkfish.js + +++ darkfish.js + @@ -109,13 +109,15 @@ function hookSearch() { + function highlightTarget( anchor ) { + console.debug( "Highlighting target '%s'.", anchor ); + + - $("a[name=" + anchor + "]").each( function() { + - if ( !$(this).parent().parent().hasClass('target-section') ) { + - console.debug( "Wrapping the target-section" ); + - $('div.method-detail').unwrap( 'div.target-section' ); + - $(this).parent().wrap( '
' ); + - } else { + - console.debug( "Already wrapped." ); + + $("a[name]").each( function() { + + if ( $(this).attr("name") == anchor ) { + + if ( !$(this).parent().parent().hasClass('target-section') ) { + + console.debug( "Wrapping the target-section" ); + + $('div.method-detail').unwrap( 'div.target-section' ); + + $(this).parent().wrap( '
' ); + + } else { + + console.debug( "Already wrapped." ); + + } + } + }); + }; + +RDoc 3.12.1 and RDoc 4.0.0.rc.2 and newer are not vulnerable to this exploit. + +This exploit was discovered by Evgeny Ermakov . + +This vulnerability has been assigned the CVE identifier CVE-2013-0256. + diff --git a/History.rdoc b/History.rdoc index f184dd74d6..85820c632a 100644 --- a/History.rdoc +++ b/History.rdoc @@ -1,3 +1,10 @@ +=== 3.12.1 + +* Bug fix + * Fixed an XSS exploit in darkfish.js. This could lead to cookie disclosure + to third parties. See CVE-2012-0256[link:CVE-2012-0256.rdoc] for full + details including a patch you can apply to generated RDoc documentation. + === 3.12 / 2011-12-15 * Minor enhancements diff --git a/Manifest.txt b/Manifest.txt index 4ace335c64..827031e8b2 100644 --- a/Manifest.txt +++ b/Manifest.txt @@ -1,5 +1,6 @@ .autotest .document +CVE-2013-0256.rdoc DEVELOPERS.rdoc History.rdoc LEGAL.rdoc diff --git a/Rakefile b/Rakefile index aad7380b54..36232e6117 100644 --- a/Rakefile +++ b/Rakefile @@ -42,6 +42,7 @@ Depending on your version of ruby, you may need to install ruby rdoc/ri data: self.testlib = :minitest self.extra_rdoc_files += %w[ DEVELOPERS.rdoc + CVE-2013-0256.rdoc History.rdoc LICENSE.rdoc LEGAL.rdoc diff --git a/lib/rdoc/generator/template/darkfish/js/darkfish.js b/lib/rdoc/generator/template/darkfish/js/darkfish.js index 4be722fac3..f26fd45d3a 100644 --- a/lib/rdoc/generator/template/darkfish/js/darkfish.js +++ b/lib/rdoc/generator/template/darkfish/js/darkfish.js @@ -109,13 +109,15 @@ function hookSearch() { function highlightTarget( anchor ) { console.debug( "Highlighting target '%s'.", anchor ); - $("a[name=" + anchor + "]").each( function() { - if ( !$(this).parent().parent().hasClass('target-section') ) { - console.debug( "Wrapping the target-section" ); - $('div.method-detail').unwrap( 'div.target-section' ); - $(this).parent().wrap( '
' ); - } else { - console.debug( "Already wrapped." ); + $("a[name]").each( function() { + if ( $(this).attr("name") == anchor ) { + if ( !$(this).parent().parent().hasClass('target-section') ) { + console.debug( "Wrapping the target-section" ); + $('div.method-detail').unwrap( 'div.target-section' ); + $(this).parent().wrap( '
' ); + } else { + console.debug( "Already wrapped." ); + } } }); }; From adef57aaf3aa47945f9c906419ad3eb0d6ff1269 Mon Sep 17 00:00:00 2001 From: Eric Hodel Date: Sun, 25 Nov 2012 18:12:00 -0800 Subject: [PATCH 02/20] Ensure parser files are built before checking the manifest --- Rakefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Rakefile b/Rakefile index 36232e6117..e4a19c7996 100644 --- a/Rakefile +++ b/Rakefile @@ -65,6 +65,7 @@ Depending on your version of ruby, you may need to install ruby rdoc/ri data: end task :generate => PARSER_FILES +task :check_manifest => :generate rule '.rb' => '.ry' do |t| racc = Gem.bin_path 'racc', 'racc' From 09d00bed64be9f6c73e04945595c668ea1798071 Mon Sep 17 00:00:00 2001 From: Eric Hodel Date: Mon, 4 Feb 2013 18:54:18 -0800 Subject: [PATCH 03/20] Updated Manifest with generated rd parser --- History.rdoc | 3 ++- Manifest.txt | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/History.rdoc b/History.rdoc index 85820c632a..2b9b0192e8 100644 --- a/History.rdoc +++ b/History.rdoc @@ -1,9 +1,10 @@ === 3.12.1 -* Bug fix +* Bug fixes * Fixed an XSS exploit in darkfish.js. This could lead to cookie disclosure to third parties. See CVE-2012-0256[link:CVE-2012-0256.rdoc] for full details including a patch you can apply to generated RDoc documentation. + * Ensured that rd parser files are generated before checking the manifest. === 3.12 / 2011-12-15 diff --git a/Manifest.txt b/Manifest.txt index 827031e8b2..5f2f9528d5 100644 --- a/Manifest.txt +++ b/Manifest.txt @@ -129,8 +129,10 @@ lib/rdoc/parser/ruby_tools.rb lib/rdoc/parser/simple.rb lib/rdoc/parser/text.rb lib/rdoc/rd.rb +lib/rdoc/rd/block_parser.rb lib/rdoc/rd/block_parser.ry lib/rdoc/rd/inline.rb +lib/rdoc/rd/inline_parser.rb lib/rdoc/rd/inline_parser.ry lib/rdoc/rdoc.rb lib/rdoc/require.rb From 2f35a52a8e51546713f13e7417c6bd8ac9dd4916 Mon Sep 17 00:00:00 2001 From: Eric Hodel Date: Tue, 5 Feb 2013 20:01:30 -0800 Subject: [PATCH 04/20] Update History with correct CVE Add rdoc 3.9.5 to unaffected versions --- CVE-2013-0256.rdoc | 3 ++- History.rdoc | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CVE-2013-0256.rdoc b/CVE-2013-0256.rdoc index 79acbe86ac..b285b6ac45 100644 --- a/CVE-2013-0256.rdoc +++ b/CVE-2013-0256.rdoc @@ -40,7 +40,8 @@ patch will correct all affected versions: }); }; -RDoc 3.12.1 and RDoc 4.0.0.rc.2 and newer are not vulnerable to this exploit. +RDoc 3.9.5, 3.12.1 and RDoc 4.0.0.rc.2 and newer are not vulnerable to this +exploit. This exploit was discovered by Evgeny Ermakov . diff --git a/History.rdoc b/History.rdoc index 2b9b0192e8..e522db6f59 100644 --- a/History.rdoc +++ b/History.rdoc @@ -2,7 +2,7 @@ * Bug fixes * Fixed an XSS exploit in darkfish.js. This could lead to cookie disclosure - to third parties. See CVE-2012-0256[link:CVE-2012-0256.rdoc] for full + to third parties. See CVE-2013-0256[rdoc-ref:CVE-2013-0256.rdoc] for full details including a patch you can apply to generated RDoc documentation. * Ensured that rd parser files are generated before checking the manifest. From 7132b320e7d4ef8f9bd851edf747fb6c62ac57be Mon Sep 17 00:00:00 2001 From: Eric Hodel Date: Tue, 5 Feb 2013 20:09:56 -0800 Subject: [PATCH 05/20] Set release date --- History.rdoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/History.rdoc b/History.rdoc index e522db6f59..29daa38a8e 100644 --- a/History.rdoc +++ b/History.rdoc @@ -1,4 +1,4 @@ -=== 3.12.1 +=== 3.12.1 / 2013-02-05 * Bug fixes * Fixed an XSS exploit in darkfish.js. This could lead to cookie disclosure From 04afc969b1a87ff4f0c60bb6c03863a87f010861 Mon Sep 17 00:00:00 2001 From: Eric Hodel Date: Wed, 6 Feb 2013 00:11:39 -0800 Subject: [PATCH 06/20] Update version --- lib/rdoc.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/rdoc.rb b/lib/rdoc.rb index 86a4ecf5f2..9928325a4a 100644 --- a/lib/rdoc.rb +++ b/lib/rdoc.rb @@ -108,7 +108,7 @@ def self.const_missing const_name # :nodoc: ## # RDoc version you are using - VERSION = '3.12' + VERSION = '3.12.1' ## # Method visibilities From c24963cafe9d591b672926a7ce4720187eb37cc0 Mon Sep 17 00:00:00 2001 From: Eric Hodel Date: Fri, 1 Jun 2012 16:03:51 -0700 Subject: [PATCH 07/20] Fixed syntax highlighting of regular expressions In HTML output, syntax highlighting of regular expressions could become corrupted. This patch whitelists "/" as a character with a matching terminal like '"' and "'" for strings. See also Ruby Bug #6488 and #6523 Conflicts: History.rdoc --- History.rdoc | 6 ++++++ lib/rdoc/ruby_lex.rb | 2 +- test/test_rdoc_ruby_lex.rb | 22 ++++++++++++++++++++++ 3 files changed, 29 insertions(+), 1 deletion(-) diff --git a/History.rdoc b/History.rdoc index 29daa38a8e..39400349bf 100644 --- a/History.rdoc +++ b/History.rdoc @@ -1,3 +1,9 @@ +=== 3.13 + +* Bug fixes + * Fixed bug in syntax-highlighting that would corrupt regular expressions. + Ruby Bug #6488 by Benny Lyne Amorsen. + === 3.12.1 / 2013-02-05 * Bug fixes diff --git a/lib/rdoc/ruby_lex.rb b/lib/rdoc/ruby_lex.rb index 209f01133f..285d3d8c69 100644 --- a/lib/rdoc/ruby_lex.rb +++ b/lib/rdoc/ruby_lex.rb @@ -1163,7 +1163,7 @@ def identify_string(ltype, quoted = ltype, type = nil) @ltype = ltype @quoted = quoted - str = if ltype == quoted and %w[" '].include? ltype then + str = if ltype == quoted and %w[" ' /].include? ltype then ltype.dup elsif RUBY_VERSION > '1.9' then "%#{type or PERCENT_LTYPE.key ltype}#{PERCENT_PAREN_REV[quoted]}" diff --git a/test/test_rdoc_ruby_lex.rb b/test/test_rdoc_ruby_lex.rb index 6410c587b7..078997d7e1 100644 --- a/test/test_rdoc_ruby_lex.rb +++ b/test/test_rdoc_ruby_lex.rb @@ -94,6 +94,17 @@ def test_class_tokenize_heredoc_percent_N assert_equal expected, tokens end + def test_class_tokenize_percent_r + tokens = RDoc::RubyLex.tokenize '%r[hi]', nil + + expected = [ + @TK::TkREGEXP.new( 0, 1, 0, '%r[hi]'), + @TK::TkNL .new( 6, 1, 6, "\n"), + ] + + assert_equal expected, tokens + end + def test_class_tokenize_percent_w tokens = RDoc::RubyLex.tokenize '%w[hi]', nil @@ -105,6 +116,17 @@ def test_class_tokenize_percent_w assert_equal expected, tokens end + def test_class_tokenize_regexp + tokens = RDoc::RubyLex.tokenize "/hay/", nil + + expected = [ + @TK::TkREGEXP.new( 0, 1, 0, "/hay/"), + @TK::TkNL .new( 5, 1, 5, "\n"), + ] + + assert_equal expected, tokens + end + def test_class_tokenize_string tokens = RDoc::RubyLex.tokenize "'hi'", nil From 65804516ae2c6bf957a213d21684bd31d0a554a0 Mon Sep 17 00:00:00 2001 From: Eric Hodel Date: Fri, 1 Jun 2012 16:16:45 -0700 Subject: [PATCH 08/20] Fixed Regexp backreference corruption Regular expression parsing was switched to use single-quote escaping rules. Using double quote rules, extra backslashes and white spaces was added. See also Ruby Bug #6488 and #6523 --- lib/rdoc/ruby_lex.rb | 14 ++++++++------ test/test_rdoc_ruby_lex.rb | 11 +++++++++++ 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/lib/rdoc/ruby_lex.rb b/lib/rdoc/ruby_lex.rb index 285d3d8c69..3d4d6a27ef 100644 --- a/lib/rdoc/ruby_lex.rb +++ b/lib/rdoc/ruby_lex.rb @@ -1189,14 +1189,16 @@ def identify_string(ltype, quoted = ltype, type = nil) else ungetc end - elsif ch == '\\' and @ltype == "'" #' - case ch = getc - when "\\", "\n", "'" + elsif ch == '\\' + if %w[' /].include? @ltype then + case ch = getc + when "\\", "\n", "'" + else + ungetc + end else - ungetc + str << read_escape end - elsif ch == '\\' #' - str << read_escape end if close then diff --git a/test/test_rdoc_ruby_lex.rb b/test/test_rdoc_ruby_lex.rb index 078997d7e1..8c942d47dc 100644 --- a/test/test_rdoc_ruby_lex.rb +++ b/test/test_rdoc_ruby_lex.rb @@ -127,6 +127,17 @@ def test_class_tokenize_regexp assert_equal expected, tokens end + def test_class_tokenize_regexp_backref + tokens = RDoc::RubyLex.tokenize "/[csh](..) [csh]\\1 in/", nil + + expected = [ + @TK::TkREGEXP.new( 0, 1, 0, "/[csh](..) [csh]\\1 in/"), + @TK::TkNL .new(22, 1, 22, "\n"), + ] + + assert_equal expected, tokens + end + def test_class_tokenize_string tokens = RDoc::RubyLex.tokenize "'hi'", nil From 26fc7904ce76894b2d500aef092cc465d49fb6aa Mon Sep 17 00:00:00 2001 From: Eric Hodel Date: Thu, 2 Aug 2012 14:55:48 -0700 Subject: [PATCH 09/20] Fixed lexing of character syntax like ?z Character syntax `?z` was lexed as an integer token `"zz"`. This has been fixed to lex as a single character string `"z"`. While this won't round-trip, it is acceptable for syntax highlighting. This was discovered by Xavier Noria (@fxn) in rails/rails#7172. Conflicts: History.rdoc --- History.rdoc | 2 ++ lib/rdoc/ruby_lex.rb | 7 +------ test/test_rdoc_ruby_lex.rb | 11 +++++++++++ 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/History.rdoc b/History.rdoc index 39400349bf..9f2fb9e58e 100644 --- a/History.rdoc +++ b/History.rdoc @@ -3,6 +3,8 @@ * Bug fixes * Fixed bug in syntax-highlighting that would corrupt regular expressions. Ruby Bug #6488 by Benny Lyne Amorsen. + * Fixed lexing of character syntax (?x). Reported by Xavier + Noria. === 3.12.1 / 2013-02-05 diff --git a/lib/rdoc/ruby_lex.rb b/lib/rdoc/ruby_lex.rb index 3d4d6a27ef..afb265156b 100644 --- a/lib/rdoc/ruby_lex.rb +++ b/lib/rdoc/ruby_lex.rb @@ -536,13 +536,8 @@ def lex_init() @lex_state = EXPR_BEG; Token(TkQUESTION) else - str = ch - if ch == '\\' - str << read_escape - end @lex_state = EXPR_END - str << (ch.respond_to?(:ord) ? ch.ord : ch[0]) - Token(TkINTEGER, str) + Token(TkSTRING, ch) end end end diff --git a/test/test_rdoc_ruby_lex.rb b/test/test_rdoc_ruby_lex.rb index 8c942d47dc..7e920286fd 100644 --- a/test/test_rdoc_ruby_lex.rb +++ b/test/test_rdoc_ruby_lex.rb @@ -30,6 +30,17 @@ def test_class_tokenize assert_equal expected, tokens end + def test_class_tokenize_character_literal + tokens = RDoc::RubyLex.tokenize "?\\", nil + + expected = [ + @TK::TkSTRING.new( 0, 1, 0, "\\"), + @TK::TkNL .new( 2, 1, 2, "\n"), + ] + + assert_equal expected, tokens + end + def test_class_tokenize_def_heredoc tokens = RDoc::RubyLex.tokenize <<-'RUBY', nil def x From 8eb2ae25b0444f4fcff81ab04e1a1c7c838b5ca8 Mon Sep 17 00:00:00 2001 From: Eric Hodel Date: Sun, 25 Nov 2012 15:50:54 -0800 Subject: [PATCH 10/20] Fixed parsing of escaped characters in strings --- lib/rdoc/ruby_lex.rb | 2 ++ test/test_rdoc_ruby_lex.rb | 11 +++++++++++ 2 files changed, 13 insertions(+) diff --git a/lib/rdoc/ruby_lex.rb b/lib/rdoc/ruby_lex.rb index afb265156b..4f623eb375 100644 --- a/lib/rdoc/ruby_lex.rb +++ b/lib/rdoc/ruby_lex.rb @@ -1188,6 +1188,8 @@ def identify_string(ltype, quoted = ltype, type = nil) if %w[' /].include? @ltype then case ch = getc when "\\", "\n", "'" + when @ltype + str << ch else ungetc end diff --git a/test/test_rdoc_ruby_lex.rb b/test/test_rdoc_ruby_lex.rb index 7e920286fd..13ebe8c47d 100644 --- a/test/test_rdoc_ruby_lex.rb +++ b/test/test_rdoc_ruby_lex.rb @@ -149,6 +149,17 @@ def test_class_tokenize_regexp_backref assert_equal expected, tokens end + def test_class_tokenize_regexp_escape + tokens = RDoc::RubyLex.tokenize "/\\//", nil + + expected = [ + @TK::TkREGEXP.new( 0, 1, 0, "/\\//"), + @TK::TkNL .new( 4, 1, 4, "\n"), + ] + + assert_equal expected, tokens + end + def test_class_tokenize_string tokens = RDoc::RubyLex.tokenize "'hi'", nil From 976d31fcc330e0d5e5c64e74dfc8cce9d8848088 Mon Sep 17 00:00:00 2001 From: Eric Hodel Date: Sun, 25 Nov 2012 16:19:46 -0800 Subject: [PATCH 11/20] Fixed tokenization of "a%1" Updated failed tokenization examples to use bad octal digits Conflicts: History.rdoc test/test_rdoc_markup_to_html.rb test/test_rdoc_markup_to_html_snippet.rb --- History.rdoc | 1 + lib/rdoc/ruby_lex.rb | 2 +- test/test_rdoc_markup_to_html_snippet.rb | 8 +++++--- test/test_rdoc_ruby_lex.rb | 15 +++++++++++++++ 4 files changed, 22 insertions(+), 4 deletions(-) diff --git a/History.rdoc b/History.rdoc index 9f2fb9e58e..c4503f322a 100644 --- a/History.rdoc +++ b/History.rdoc @@ -5,6 +5,7 @@ Ruby Bug #6488 by Benny Lyne Amorsen. * Fixed lexing of character syntax (?x). Reported by Xavier Noria. + * Fixed tokenization of % when it is not followed by a $-string type === 3.12.1 / 2013-02-05 diff --git a/lib/rdoc/ruby_lex.rb b/lib/rdoc/ruby_lex.rb index 4f623eb375..e91cf46c10 100644 --- a/lib/rdoc/ruby_lex.rb +++ b/lib/rdoc/ruby_lex.rb @@ -1034,7 +1034,7 @@ def identify_quotation type = nil lt = "\"" else - raise Error, "unknown type of %string #{type.inspect}" + return Token(TkMOD, '%') end # if ch !~ /\W/ # ungetc diff --git a/test/test_rdoc_markup_to_html_snippet.rb b/test/test_rdoc_markup_to_html_snippet.rb index 067ac9647b..2f48469190 100644 --- a/test/test_rdoc_markup_to_html_snippet.rb +++ b/test/test_rdoc_markup_to_html_snippet.rb @@ -379,19 +379,21 @@ def test_accept_verbatim_ruby_error rdoc.options = options RDoc::RDoc.current = rdoc - verb = @RM::Verbatim.new("a %z'foo' # => blah\n") + verb = @RM::Verbatim.new("a % 09 # => blah\n") @to.start_accepting @to.accept_verbatim verb + inner = CGI.escapeHTML "a % 09 # => blah" + expected = <<-EXPECTED -
a %z'foo' # => blah
+
a % 09 # => blah
 

     EXPECTED
 
     assert_equal expected, @to.res.join
-    assert_equal 19, @to.characters
+    assert_equal 16, @to.characters
   end
 
   def test_add_paragraph
diff --git a/test/test_rdoc_ruby_lex.rb b/test/test_rdoc_ruby_lex.rb
index 13ebe8c47d..08662a945c 100644
--- a/test/test_rdoc_ruby_lex.rb
+++ b/test/test_rdoc_ruby_lex.rb
@@ -105,6 +105,21 @@ def test_class_tokenize_heredoc_percent_N
     assert_equal expected, tokens
   end
 
+  def test_class_tokenize_percent_1
+    tokens = RDoc::RubyLex.tokenize 'v%10==10', nil
+
+    expected = [
+      @TK::TkIDENTIFIER.new(0, 1, 0, 'v'),
+      @TK::TkMOD.new(       1, 1, 1, '%'),
+      @TK::TkINTEGER.new(   2, 1, 2, '10'),
+      @TK::TkEQ.new(        4, 1, 4, '=='),
+      @TK::TkINTEGER.new(   6, 1, 6, '10'),
+      @TK::TkNL.new(        8, 1, 8, "\n"),
+    ]
+
+    assert_equal expected, tokens
+  end
+
   def test_class_tokenize_percent_r
     tokens = RDoc::RubyLex.tokenize '%r[hi]', nil
 

From 24efb019aa4ac19bd0221dcc01d5c14d88374119 Mon Sep 17 00:00:00 2001
From: Eric Hodel 
Date: Sun, 25 Nov 2012 17:15:31 -0800
Subject: [PATCH 12/20] Gave TkEND_OF_SCRIPT#text a value

This allows __END__ to be shown in HTML verbatim blocks

Conflicts:
	History.rdoc
---
 History.rdoc               |  2 ++
 lib/rdoc/ruby_lex.rb       |  5 +++--
 test/test_rdoc_ruby_lex.rb | 11 +++++++++++
 3 files changed, 16 insertions(+), 2 deletions(-)

diff --git a/History.rdoc b/History.rdoc
index c4503f322a..6e34ba06af 100644
--- a/History.rdoc
+++ b/History.rdoc
@@ -6,6 +6,8 @@
   * Fixed lexing of character syntax (?x).  Reported by Xavier
     Noria.
   * Fixed tokenization of % when it is not followed by a $-string type
+  * Fixed display of __END__ in documentation examples in HTML output
+
 
 === 3.12.1 / 2013-02-05
 
diff --git a/lib/rdoc/ruby_lex.rb b/lib/rdoc/ruby_lex.rb
index e91cf46c10..f3cb0046a0 100644
--- a/lib/rdoc/ruby_lex.rb
+++ b/lib/rdoc/ruby_lex.rb
@@ -412,7 +412,7 @@ def token
   def lex_init()
     @OP = IRB::SLex.new
     @OP.def_rules("\0", "\004", "\032") do |op, io|
-      Token(TkEND_OF_SCRIPT)
+      Token(TkEND_OF_SCRIPT, '')
     end
 
     @OP.def_rules(" ", "\t", "\f", "\r", "\13") do |op, io|
@@ -808,7 +808,8 @@ def lex_int2
 
     @OP.def_rule("_") do
       if peek_match?(/_END__/) and @lex_state == EXPR_BEG then
-        Token(TkEND_OF_SCRIPT)
+        6.times { getc }
+        Token(TkEND_OF_SCRIPT, '__END__')
       else
         ungetc
         identify_identifier
diff --git a/test/test_rdoc_ruby_lex.rb b/test/test_rdoc_ruby_lex.rb
index 08662a945c..4029c1472a 100644
--- a/test/test_rdoc_ruby_lex.rb
+++ b/test/test_rdoc_ruby_lex.rb
@@ -30,6 +30,17 @@ def test_class_tokenize
     assert_equal expected, tokens
   end
 
+  def test_class_tokenize___END__
+    tokens = RDoc::RubyLex.tokenize '__END__', nil
+
+    expected = [
+      @TK::TkEND_OF_SCRIPT.new(0, 1, 0, '__END__'),
+      @TK::TkNL           .new(7, 1, 7, "\n"),
+    ]
+
+    assert_equal expected, tokens
+  end
+
   def test_class_tokenize_character_literal
     tokens = RDoc::RubyLex.tokenize "?\\", nil
 

From 55448537fec9940970620fcb447dc873567f6e98 Mon Sep 17 00:00:00 2001
From: Eric Hodel 
Date: Sun, 25 Nov 2012 18:39:21 -0800
Subject: [PATCH 13/20] Tokenize out-of-place reserved words as identifiers

---
 History.rdoc               |  1 +
 lib/rdoc/ruby_lex.rb       |  2 ++
 test/test_rdoc_ruby_lex.rb | 17 +++++++++++++++++
 3 files changed, 20 insertions(+)

diff --git a/History.rdoc b/History.rdoc
index 6e34ba06af..e21b1d2b7e 100644
--- a/History.rdoc
+++ b/History.rdoc
@@ -7,6 +7,7 @@
     Noria.
   * Fixed tokenization of % when it is not followed by a $-string type
   * Fixed display of __END__ in documentation examples in HTML output
+  * Fixed tokenization of reserved words used as new-style hash keys
 
 
 === 3.12.1 / 2013-02-05
diff --git a/lib/rdoc/ruby_lex.rb b/lib/rdoc/ruby_lex.rb
index f3cb0046a0..845569b0bc 100644
--- a/lib/rdoc/ruby_lex.rb
+++ b/lib/rdoc/ruby_lex.rb
@@ -936,6 +936,8 @@ def identify_identifier
                   @indent += 1
                   @indent_stack.push token_c
                 end
+              else
+                token_c = TkIDENTIFIER
               end
 
             elsif DEINDENT_CLAUSE.include?(token)
diff --git a/test/test_rdoc_ruby_lex.rb b/test/test_rdoc_ruby_lex.rb
index 4029c1472a..1dc11e95a3 100644
--- a/test/test_rdoc_ruby_lex.rb
+++ b/test/test_rdoc_ruby_lex.rb
@@ -77,6 +77,23 @@ def x
     assert_equal expected, tokens
   end
 
+  def test_class_tokenize_hash_symbol
+    tokens = RDoc::RubyLex.tokenize '{ class:"foo" }', nil
+
+    expected = [
+      @TK::TkLBRACE    .new( 0, 1,  0, '{'),
+      @TK::TkSPACE     .new( 1, 1,  1, ' '),
+      @TK::TkIDENTIFIER.new( 2, 1,  2, 'class'),
+      @TK::TkSYMBEG    .new( 7, 1,  7, ':'),
+      @TK::TkSTRING    .new( 8, 1,  8, '"foo"'),
+      @TK::TkSPACE     .new(13, 1, 13, ' '),
+      @TK::TkRBRACE    .new(14, 1, 14, '}'),
+      @TK::TkNL        .new(15, 1, 15, "\n"),
+    ]
+
+    assert_equal expected, tokens
+  end
+
   def test_class_tokenize_heredoc_indent
     tokens = RDoc::RubyLex.tokenize <<-'RUBY', nil
 string = <<-STRING

From 78ef23e47b1acc5f19fb8c1ee9664dadb0ec8d8e Mon Sep 17 00:00:00 2001
From: Eric Hodel 
Date: Sat, 8 Dec 2012 13:06:12 -0800
Subject: [PATCH 14/20] Parse unicode characters above \uFFFF
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The regular expression matching identifiers was incomplete for unicode
characters.  Now 𝖒 can be parsed in an identifier.

Ruby Bug #7524
---
 lib/rdoc/ruby_lex.rb       |  2 +-
 test/test_rdoc_ruby_lex.rb | 10 ++++++++++
 2 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/lib/rdoc/ruby_lex.rb b/lib/rdoc/ruby_lex.rb
index 845569b0bc..313e69ea47 100644
--- a/lib/rdoc/ruby_lex.rb
+++ b/lib/rdoc/ruby_lex.rb
@@ -857,7 +857,7 @@ def identify_gvar
   end
 
   IDENT_RE = if defined? Encoding then
-               /[\w\u0080-\uFFFF]/u
+               eval '/[\w\u{0080}-\u{FFFFF}]/u' # 1.8 can't parse \u{}
              else
                /[\w\x80-\xFF]/
              end
diff --git a/test/test_rdoc_ruby_lex.rb b/test/test_rdoc_ruby_lex.rb
index 1dc11e95a3..dfa350e018 100644
--- a/test/test_rdoc_ruby_lex.rb
+++ b/test/test_rdoc_ruby_lex.rb
@@ -1,3 +1,5 @@
+# coding: UTF-8
+
 require 'rdoc/test_case'
 
 class TestRDocRubyLex < RDoc::TestCase
@@ -133,6 +135,14 @@ def test_class_tokenize_heredoc_percent_N
     assert_equal expected, tokens
   end
 
+  def test_class_tokenize_identifier_high_unicode
+    tokens = RDoc::RubyLex.tokenize '𝖒', nil
+
+    expected = @TK::TkIDENTIFIER.new(0, 1, 0, '𝖒')
+
+    assert_equal expected, tokens.first
+  end
+
   def test_class_tokenize_percent_1
     tokens = RDoc::RubyLex.tokenize 'v%10==10', nil
 

From a5fc3d63fe27d7144629077fb0fdbaaf31559a30 Mon Sep 17 00:00:00 2001
From: Eric Hodel 
Date: Mon, 17 Dec 2012 23:27:47 -0800
Subject: [PATCH 15/20] Added limited HEREDOC tokenization

A TkHEREDOC is only generated when the heredoc identifier is followed by
a line end.  When followed by a method call, comma, etc. a TkSTRING is
returned instead so the source output in the HTML view is correct.

Fixes #162

Conflicts:
	History.rdoc
	lib/rdoc/store.rb
---
 History.rdoc               |  4 ++-
 lib/rdoc/ruby_lex.rb       | 24 +++++++++++++---
 lib/rdoc/ruby_token.rb     |  1 +
 test/test_rdoc_ruby_lex.rb | 56 ++++++++++++++++++++++++++++++++++----
 4 files changed, 75 insertions(+), 10 deletions(-)

diff --git a/History.rdoc b/History.rdoc
index e21b1d2b7e..2f328afec0 100644
--- a/History.rdoc
+++ b/History.rdoc
@@ -8,7 +8,9 @@
   * Fixed tokenization of % when it is not followed by a $-string type
   * Fixed display of __END__ in documentation examples in HTML output
   * Fixed tokenization of reserved words used as new-style hash keys
-
+  * Fixed HEREDOC output for the limited case of a heredoc followed by a line
+    end.  When a HEREDOC is not followed by a line end RDoc is not currently
+    smart enough to restore the source correctly.  Bug #162 by Zachary Scott.
 
 === 3.12.1 / 2013-02-05
 
diff --git a/lib/rdoc/ruby_lex.rb b/lib/rdoc/ruby_lex.rb
index 313e69ea47..e6e0b41aab 100644
--- a/lib/rdoc/ruby_lex.rb
+++ b/lib/rdoc/ruby_lex.rb
@@ -982,12 +982,13 @@ def identify_here_document
       indent = true
     end
     if /['"`]/ =~ ch
-      lt = ch
+      user_quote = lt = ch
       quoted = ""
       while (c = getc) && c != lt
         quoted.concat c
       end
     else
+      user_quote = nil
       lt = '"'
       quoted = ch.dup
       while (c = getc) && c =~ /\w/
@@ -1007,8 +1008,17 @@ def identify_here_document
       end
     end
 
+    output_heredoc = reserve.join =~ /\A\r?\n\z/
+
+    if output_heredoc then
+      doc = '<<'
+      doc << '-' if indent
+      doc << "#{user_quote}#{quoted}#{user_quote}\n"
+    else
+      doc = '"'
+    end
+
     @here_header = false
-    doc = '"'
     while l = gets
       l = l.sub(/(:?\r)?\n\z/, "\n")
       if (indent ? l.strip : l.chomp) == quoted
@@ -1016,7 +1026,12 @@ def identify_here_document
       end
       doc << l
     end
-    doc << '"'
+
+    if output_heredoc then
+      doc << l.chomp
+    else
+      doc << '"'
+    end
 
     @here_header = true
     @here_readed.concat reserve
@@ -1024,9 +1039,10 @@ def identify_here_document
       ungetc ch
     end
 
+    token_class = output_heredoc ? RDoc::RubyLex::TkHEREDOC : Ltype2Token[lt]
     @ltype = ltback
     @lex_state = EXPR_END
-    Token(Ltype2Token[lt], doc)
+    Token(token_class, doc)
   end
 
   def identify_quotation
diff --git a/lib/rdoc/ruby_token.rb b/lib/rdoc/ruby_token.rb
index 687ef2fe80..d3333bc6a1 100644
--- a/lib/rdoc/ruby_token.rb
+++ b/lib/rdoc/ruby_token.rb
@@ -331,6 +331,7 @@ def Token(token, value = nil)
     [:TkINTEGER,    TkVal],
     [:TkFLOAT,      TkVal],
     [:TkSTRING,     TkVal],
+    [:TkHEREDOC,    TkVal],
     [:TkXSTRING,    TkVal],
     [:TkREGEXP,     TkVal],
     [:TkSYMBOL,     TkVal],
diff --git a/test/test_rdoc_ruby_lex.rb b/test/test_rdoc_ruby_lex.rb
index dfa350e018..0dcb42565d 100644
--- a/test/test_rdoc_ruby_lex.rb
+++ b/test/test_rdoc_ruby_lex.rb
@@ -70,7 +70,8 @@ def x
       @TK::TkIDENTIFIER.new( 4, 1,  4, 'x'),
       @TK::TkNL        .new( 5, 1,  5, "\n"),
       @TK::TkSPACE     .new( 6, 2,  0, '  '),
-      @TK::TkSTRING    .new( 8, 2,  2, %Q{"Line 1\nLine 2\n"}),
+      @TK::TkHEREDOC   .new( 8, 2,  2,
+                            %Q{<
Date: Sun, 24 Feb 2013 11:25:08 -0800
Subject: [PATCH 16/20] Fixed tests broken by incompetent cherry-picking

Since I didn't run tests during cherry pick I missed some merges.  These
have no been corrected.
---
 test/test_rdoc_markup_to_html.rb         | 4 ++--
 test/test_rdoc_markup_to_html_snippet.rb | 4 +---
 2 files changed, 3 insertions(+), 5 deletions(-)

diff --git a/test/test_rdoc_markup_to_html.rb b/test/test_rdoc_markup_to_html.rb
index 88acf57ea8..a84270fbb5 100644
--- a/test/test_rdoc_markup_to_html.rb
+++ b/test/test_rdoc_markup_to_html.rb
@@ -350,14 +350,14 @@ def test_accept_verbatim_parseable_error
     rdoc.options = options
     RDoc::RDoc.current = rdoc
 
-    verb = @RM::Verbatim.new("a %z'foo' # => blah\n")
+    verb = @RM::Verbatim.new("a % 09 # => blah\n")
 
     @to.start_accepting
     @to.accept_verbatim verb
 
     expected = <<-EXPECTED
 
-
a %z'foo' # => blah
+
a % 09 # => blah
 

     EXPECTED
 
diff --git a/test/test_rdoc_markup_to_html_snippet.rb b/test/test_rdoc_markup_to_html_snippet.rb
index 2f48469190..38257a89b1 100644
--- a/test/test_rdoc_markup_to_html_snippet.rb
+++ b/test/test_rdoc_markup_to_html_snippet.rb
@@ -384,8 +384,6 @@ def test_accept_verbatim_ruby_error
     @to.start_accepting
     @to.accept_verbatim verb
 
-    inner = CGI.escapeHTML "a % 09 # => blah"
-
     expected = <<-EXPECTED
 
 
a % 09 # => blah
@@ -549,7 +547,7 @@ def test_convert_limit_verbatim_multiline
 
Look for directives in a normal comment block:
 
 
# :stopdoc:
-# Don't display comment from this point forward

+# Don't display comment from this point forward

     EXPECTED
 
     actual = @to.convert rdoc

From 92c591cc61581c8d55e3408ae35f72848759b0be Mon Sep 17 00:00:00 2001
From: Eric Hodel 
Date: Sun, 24 Feb 2013 11:27:02 -0800
Subject: [PATCH 17/20] Set version to 3.12.2

The parser changes to support %i were only bug fixes to master.

As it happens, %i support is by accident and RDoc consideres this a
method call to %.  This should probably be fixed in master at some point.
---
 History.rdoc | 2 +-
 lib/rdoc.rb  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/History.rdoc b/History.rdoc
index 2f328afec0..50c1edebf3 100644
--- a/History.rdoc
+++ b/History.rdoc
@@ -1,4 +1,4 @@
-=== 3.13
+=== 3.12.2
 
 * Bug fixes
   * Fixed bug in syntax-highlighting that would corrupt regular expressions.
diff --git a/lib/rdoc.rb b/lib/rdoc.rb
index 9928325a4a..b03f8270f8 100644
--- a/lib/rdoc.rb
+++ b/lib/rdoc.rb
@@ -108,7 +108,7 @@ def self.const_missing const_name # :nodoc:
   ##
   # RDoc version you are using
 
-  VERSION = '3.12.1'
+  VERSION = '3.12.2'
 
   ##
   # Method visibilities

From 751f7acf18430fc8987e38fb62bdda417139aae9 Mon Sep 17 00:00:00 2001
From: Eric Hodel 
Date: Sun, 24 Feb 2013 22:16:38 -0800
Subject: [PATCH 18/20] Fixed .travis.yml by using hoe-travis

---
 .travis.yml | 21 +++++++++++++--------
 1 file changed, 13 insertions(+), 8 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 2dbb540890..de76b7a47d 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,11 +1,16 @@
+---
+after_script:
+- rake travis:after -t
 before_script:
-  - gem install hoe
-  - rake check_extra_deps
-rvm:
-  - 1.8.7
-  - 1.9.2
-  - 1.9.3
-  - ruby-head
+- gem install hoe-travis --no-rdoc --no-ri
+- rake travis:before -t
+language: ruby
 notifications:
   email:
-    - drbrain@segment7.net
+  - drbrain@segment7.net
+rvm:
+- 1.8.7
+- 1.9.2
+- 1.9.3
+- 2.0.0
+script: rake travis

From 13f339a4b26e8d561676b09d0a7ffd895c2220ab Mon Sep 17 00:00:00 2001
From: Eric Hodel 
Date: Sun, 24 Feb 2013 22:18:10 -0800
Subject: [PATCH 19/20] Added 3.12.2 release date to history

---
 History.rdoc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/History.rdoc b/History.rdoc
index 50c1edebf3..f0a5ae9b68 100644
--- a/History.rdoc
+++ b/History.rdoc
@@ -1,4 +1,4 @@
-=== 3.12.2
+=== 3.12.2 / 2013-02-24
 
 * Bug fixes
   * Fixed bug in syntax-highlighting that would corrupt regular expressions.

From e210ead8de2f0c4e2e355d1f9c89f51b301842c9 Mon Sep 17 00:00:00 2001
From: Eric Hodel 
Date: Sun, 24 Feb 2013 22:23:25 -0800
Subject: [PATCH 20/20] Fix test in a cross-ruby way

---
 test/test_rdoc_markup_to_html_snippet.rb | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/test/test_rdoc_markup_to_html_snippet.rb b/test/test_rdoc_markup_to_html_snippet.rb
index 38257a89b1..99d9cb778f 100644
--- a/test/test_rdoc_markup_to_html_snippet.rb
+++ b/test/test_rdoc_markup_to_html_snippet.rb
@@ -543,11 +543,12 @@ def test_convert_limit_verbatim_multiline
 This routine modifies its +comment+ parameter.
     RDOC
 
+    inner = CGI.escapeHTML "# Don't display comment from this point forward"
     expected = <<-EXPECTED
 
Look for directives in a normal comment block:
 
 
# :stopdoc:
-# Don't display comment from this point forward

+#{inner}

     EXPECTED
 
     actual = @to.convert rdoc