aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorRick Olson <technoweenie@gmail.com>2007-02-04 20:04:40 +0000
committerRick Olson <technoweenie@gmail.com>2007-02-04 20:04:40 +0000
commit19fbb31464da45ac0935b3db193305f18e245ba9 (patch)
tree57543807ad63d7deb7cf5deb037fc64aed710492
parent4790228bc437c6c882bb2171a429979b37f52618 (diff)
downloadrails-19fbb31464da45ac0935b3db193305f18e245ba9.tar.gz
rails-19fbb31464da45ac0935b3db193305f18e245ba9.tar.bz2
rails-19fbb31464da45ac0935b3db193305f18e245ba9.zip
Add much-needed html-scanner tests. Fixed CDATA parsing bug. [Rick]
git-svn-id: http://svn-commit.rubyonrails.org/rails/trunk@6117 5ecf4fe2-1ee6-0310-87b1-e25e094e27de
-rw-r--r--actionpack/CHANGELOG2
-rw-r--r--actionpack/lib/action_controller/vendor/html-scanner/html/node.rb4
-rw-r--r--actionpack/test/controller/html-scanner/document_test.rb104
-rw-r--r--actionpack/test/controller/html-scanner/node_test.rb69
-rw-r--r--actionpack/test/controller/html-scanner/tag_node_test.rb239
-rw-r--r--actionpack/test/controller/html-scanner/text_node_test.rb51
-rw-r--r--actionpack/test/controller/html-scanner/tokenizer_test.rb125
7 files changed, 591 insertions, 3 deletions
diff --git a/actionpack/CHANGELOG b/actionpack/CHANGELOG
index 6481bda79c..c07c5165bd 100644
--- a/actionpack/CHANGELOG
+++ b/actionpack/CHANGELOG
@@ -1,5 +1,7 @@
*SVN*
+* Add much-needed html-scanner tests. Fixed CDATA parsing bug. [Rick]
+
* improve error message for Routing for named routes. Closes #7346 [Rob Sanheim]
* Added enhanced docs to routing assertions. Closes #7359 [Rob Sanheim]
diff --git a/actionpack/lib/action_controller/vendor/html-scanner/html/node.rb b/actionpack/lib/action_controller/vendor/html-scanner/html/node.rb
index 9b7621820e..472c5b2bae 100644
--- a/actionpack/lib/action_controller/vendor/html-scanner/html/node.rb
+++ b/actionpack/lib/action_controller/vendor/html-scanner/html/node.rb
@@ -92,7 +92,6 @@ module HTML #:nodoc:
# returns non +nil+. Returns the result of the #find call that succeeded.
def find(conditions)
conditions = validate_conditions(conditions)
-
@children.each do |child|
node = child.find(conditions)
return node if node
@@ -152,7 +151,7 @@ module HTML #:nodoc:
if scanner.skip(/!\[CDATA\[/)
scanner.scan_until(/\]\]>/)
- return CDATA.new(parent, line, pos, scanner.pre_match)
+ return CDATA.new(parent, line, pos, scanner.pre_match.gsub(/<!\[CDATA\[/, ''))
end
closing = ( scanner.scan(/\//) ? :close : nil )
@@ -410,7 +409,6 @@ module HTML #:nodoc:
# :child => /hello world/ }
def match(conditions)
conditions = validate_conditions(conditions)
-
# check content of child nodes
if conditions[:content]
if children.empty?
diff --git a/actionpack/test/controller/html-scanner/document_test.rb b/actionpack/test/controller/html-scanner/document_test.rb
new file mode 100644
index 0000000000..a6ba70dde1
--- /dev/null
+++ b/actionpack/test/controller/html-scanner/document_test.rb
@@ -0,0 +1,104 @@
+require File.dirname(__FILE__) + '/../../abstract_unit'
+require 'test/unit'
+
+class DocumentTest < Test::Unit::TestCase
+ def test_handle_doctype
+ doc = nil
+ assert_nothing_raised do
+ doc = HTML::Document.new <<-HTML.strip
+ <!DOCTYPE "blah" "blah" "blah">
+ <html>
+ </html>
+ HTML
+ end
+ assert_equal 3, doc.root.children.length
+ assert_equal %{<!DOCTYPE "blah" "blah" "blah">}, doc.root.children[0].content
+ assert_match %r{\s+}m, doc.root.children[1].content
+ assert_equal "html", doc.root.children[2].name
+ end
+
+ def test_find_img
+ doc = HTML::Document.new <<-HTML.strip
+ <html>
+ <body>
+ <p><img src="hello.gif"></p>
+ </body>
+ </html>
+ HTML
+ assert doc.find(:tag=>"img", :attributes=>{"src"=>"hello.gif"})
+ end
+
+ def test_find_all
+ doc = HTML::Document.new <<-HTML.strip
+ <html>
+ <body>
+ <p class="test"><img src="hello.gif"></p>
+ <div class="foo">
+ <p class="test">something</p>
+ <p>here is <em class="test">more</em></p>
+ </div>
+ </body>
+ </html>
+ HTML
+ all = doc.find_all :attributes => { :class => "test" }
+ assert_equal 3, all.length
+ assert_equal [ "p", "p", "em" ], all.map { |n| n.name }
+ end
+
+ def test_find_with_text
+ doc = HTML::Document.new <<-HTML.strip
+ <html>
+ <body>
+ <p>Some text</p>
+ </body>
+ </html>
+ HTML
+ assert doc.find(:content => "Some text")
+ assert doc.find(:tag => "p", :child => { :content => "Some text" })
+ assert doc.find(:tag => "p", :child => "Some text")
+ assert doc.find(:tag => "p", :content => "Some text")
+ end
+
+ def test_parse_xml
+ assert_nothing_raised { HTML::Document.new("<tags><tag/></tags>", true, true) }
+ assert_nothing_raised { HTML::Document.new("<outer><link>something</link></outer>", true, true) }
+ end
+
+ def test_parse_document
+ doc = HTML::Document.new(<<-HTML)
+ <div>
+ <h2>blah</h2>
+ <table>
+ </table>
+ </div>
+ HTML
+ assert_not_nil doc.find(:tag => "div", :children => { :count => 1, :only => { :tag => "table" } })
+ end
+
+ def test_parse_cdata
+ doc = HTML::Document.new(<<-HTML)
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
+ "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml" lang="en" xml:lang="en">
+ <head>
+ <title><![CDATA[<br>]]></title>
+ </head>
+ <body>
+ <p>this document has &lt;br&gt; for a title</p>
+ </body>
+</html>
+HTML
+
+ assert_nil doc.find(:tag => "title", :descendant => { :tag => "br" })
+ assert doc.find(:tag => "title", :child => "<br>")
+ end
+
+ def test_find_empty_tag
+ doc = HTML::Document.new("<div id='map'></div>")
+ assert_nil doc.find(:tag => "div", :attributes => { :id => "map" }, :content => /./)
+ assert doc.find(:tag => "div", :attributes => { :id => "map" }, :content => /\A\Z/)
+ assert doc.find(:tag => "div", :attributes => { :id => "map" }, :content => /^$/)
+ assert doc.find(:tag => "div", :attributes => { :id => "map" }, :content => "")
+ assert doc.find(:tag => "div", :attributes => { :id => "map" }, :content => nil)
+ end
+end
diff --git a/actionpack/test/controller/html-scanner/node_test.rb b/actionpack/test/controller/html-scanner/node_test.rb
new file mode 100644
index 0000000000..1cf0a4bb67
--- /dev/null
+++ b/actionpack/test/controller/html-scanner/node_test.rb
@@ -0,0 +1,69 @@
+require File.dirname(__FILE__) + '/../../abstract_unit'
+require 'test/unit'
+
+class NodeTest < Test::Unit::TestCase
+
+ class MockNode
+ def initialize(matched, value)
+ @matched = matched
+ @value = value
+ end
+
+ def find(conditions)
+ @matched && self
+ end
+
+ def to_s
+ @value.to_s
+ end
+ end
+
+ def setup
+ @node = HTML::Node.new("parent")
+ @node.children.concat [MockNode.new(false,1), MockNode.new(true,"two"), MockNode.new(false,:three)]
+ end
+
+ def test_match
+ assert !@node.match("foo")
+ end
+
+ def test_tag
+ assert !@node.tag?
+ end
+
+ def test_to_s
+ assert_equal "1twothree", @node.to_s
+ end
+
+ def test_find
+ assert_equal "two", @node.find('blah').to_s
+ end
+
+ def test_parse_strict
+ s = "<b foo='hello'' bar='baz'>"
+ assert_raise(RuntimeError) { HTML::Node.parse(nil,0,0,s) }
+ end
+
+ def test_parse_relaxed
+ s = "<b foo='hello'' bar='baz'>"
+ node = nil
+ assert_nothing_raised { node = HTML::Node.parse(nil,0,0,s,false) }
+ assert node.attributes.has_key?("foo")
+ assert !node.attributes.has_key?("bar")
+ end
+
+ def test_to_s_with_boolean_attrs
+ s = "<b foo bar>"
+ node = HTML::Node.parse(nil,0,0,s)
+ assert node.attributes.has_key?("foo")
+ assert node.attributes.has_key?("bar")
+ assert "<b foo bar>", node.to_s
+ end
+
+ def test_parse_with_unclosed_tag
+ s = "<span onmouseover='bang'"
+ node = nil
+ assert_nothing_raised { node = HTML::Node.parse(nil,0,0,s,false) }
+ assert node.attributes.has_key?("onmouseover")
+ end
+end
diff --git a/actionpack/test/controller/html-scanner/tag_node_test.rb b/actionpack/test/controller/html-scanner/tag_node_test.rb
new file mode 100644
index 0000000000..daeada9b9c
--- /dev/null
+++ b/actionpack/test/controller/html-scanner/tag_node_test.rb
@@ -0,0 +1,239 @@
+require File.dirname(__FILE__) + '/../../abstract_unit'
+require 'test/unit'
+
+class TagNodeTest < Test::Unit::TestCase
+ def test_open_without_attributes
+ node = tag("<tag>")
+ assert_equal "tag", node.name
+ assert_equal Hash.new, node.attributes
+ assert_nil node.closing
+ end
+
+ def test_open_with_attributes
+ node = tag("<TAG1 foo=hey_ho x:bar=\"blah blah\" BAZ='blah blah blah' >")
+ assert_equal "tag1", node.name
+ assert_equal "hey_ho", node["foo"]
+ assert_equal "blah blah", node["x:bar"]
+ assert_equal "blah blah blah", node["baz"]
+ end
+
+ def test_self_closing_without_attributes
+ node = tag("<tag/>")
+ assert_equal "tag", node.name
+ assert_equal Hash.new, node.attributes
+ assert_equal :self, node.closing
+ end
+
+ def test_self_closing_with_attributes
+ node = tag("<tag a=b/>")
+ assert_equal "tag", node.name
+ assert_equal( { "a" => "b" }, node.attributes )
+ assert_equal :self, node.closing
+ end
+
+ def test_closing_without_attributes
+ node = tag("</tag>")
+ assert_equal "tag", node.name
+ assert_nil node.attributes
+ assert_equal :close, node.closing
+ end
+
+ def test_bracket_op_when_no_attributes
+ node = tag("</tag>")
+ assert_nil node["foo"]
+ end
+
+ def test_bracket_op_when_attributes
+ node = tag("<tag a=b/>")
+ assert_equal "b", node["a"]
+ end
+
+ def test_attributes_with_escaped_quotes
+ node = tag("<tag a='b\\'c' b=\"bob \\\"float\\\"\">")
+ assert_equal "b\\'c", node["a"]
+ assert_equal "bob \\\"float\\\"", node["b"]
+ end
+
+ def test_to_s
+ node = tag("<a b=c d='f' g=\"h 'i'\" />")
+ assert_equal %(<a b='c' d='f' g='h \\'i\\'' />), node.to_s
+ end
+
+ def test_tag
+ assert tag("<tag>").tag?
+ end
+
+ def test_match_tag_as_string
+ assert tag("<tag>").match(:tag => "tag")
+ assert !tag("<tag>").match(:tag => "b")
+ end
+
+ def test_match_tag_as_regexp
+ assert tag("<tag>").match(:tag => /t.g/)
+ assert !tag("<tag>").match(:tag => /t[bqs]g/)
+ end
+
+ def test_match_attributes_as_string
+ t = tag("<tag a=something b=else />")
+ assert t.match(:attributes => {"a" => "something"})
+ assert t.match(:attributes => {"b" => "else"})
+ end
+
+ def test_match_attributes_as_regexp
+ t = tag("<tag a=something b=else />")
+ assert t.match(:attributes => {"a" => /^something$/})
+ assert t.match(:attributes => {"b" => /e.*e/})
+ assert t.match(:attributes => {"a" => /me..i/, "b" => /.ls.$/})
+ end
+
+ def test_match_attributes_as_number
+ t = tag("<tag a=15 b=3.1415 />")
+ assert t.match(:attributes => {"a" => 15})
+ assert t.match(:attributes => {"b" => 3.1415})
+ assert t.match(:attributes => {"a" => 15, "b" => 3.1415})
+ end
+
+ def test_match_attributes_exist
+ t = tag("<tag a=15 b=3.1415 />")
+ assert t.match(:attributes => {"a" => true})
+ assert t.match(:attributes => {"b" => true})
+ assert t.match(:attributes => {"a" => true, "b" => true})
+ end
+
+ def test_match_attributes_not_exist
+ t = tag("<tag a=15 b=3.1415 />")
+ assert t.match(:attributes => {"c" => false})
+ assert t.match(:attributes => {"c" => nil})
+ assert t.match(:attributes => {"a" => true, "c" => false})
+ end
+
+ def test_match_parent_success
+ t = tag("<tag a=15 b='hello'>", tag("<foo k='value'>"))
+ assert t.match(:parent => {:tag => "foo", :attributes => {"k" => /v.l/, "j" => false}})
+ end
+
+ def test_match_parent_fail
+ t = tag("<tag a=15 b='hello'>", tag("<foo k='value'>"))
+ assert !t.match(:parent => {:tag => /kafka/})
+ end
+
+ def test_match_child_success
+ t = tag("<tag x:k='something'>")
+ tag("<child v=john a=kelly>", t)
+ tag("<sib m=vaughn v=james>", t)
+ assert t.match(:child => { :tag => "sib", :attributes => {"v" => /j/}})
+ assert t.match(:child => { :attributes => {"a" => "kelly"}})
+ end
+
+ def test_match_child_fail
+ t = tag("<tag x:k='something'>")
+ tag("<child v=john a=kelly>", t)
+ tag("<sib m=vaughn v=james>", t)
+ assert !t.match(:child => { :tag => "sib", :attributes => {"v" => /r/}})
+ assert !t.match(:child => { :attributes => {"v" => false}})
+ end
+
+ def test_match_ancestor_success
+ t = tag("<tag x:k='something'>", tag("<parent v=john a=kelly>", tag("<grandparent m=vaughn v=james>")))
+ assert t.match(:ancestor => {:tag => "parent", :attributes => {"a" => /ll/}})
+ assert t.match(:ancestor => {:attributes => {"m" => "vaughn"}})
+ end
+
+ def test_match_ancestor_fail
+ t = tag("<tag x:k='something'>", tag("<parent v=john a=kelly>", tag("<grandparent m=vaughn v=james>")))
+ assert !t.match(:ancestor => {:tag => /^parent/, :attributes => {"v" => /m/}})
+ assert !t.match(:ancestor => {:attributes => {"v" => false}})
+ end
+
+ def test_match_descendant_success
+ tag("<grandchild m=vaughn v=james>", tag("<child v=john a=kelly>", t = tag("<tag x:k='something'>")))
+ assert t.match(:descendant => {:tag => "child", :attributes => {"a" => /ll/}})
+ assert t.match(:descendant => {:attributes => {"m" => "vaughn"}})
+ end
+
+ def test_match_descendant_fail
+ tag("<grandchild m=vaughn v=james>", tag("<child v=john a=kelly>", t = tag("<tag x:k='something'>")))
+ assert !t.match(:descendant => {:tag => /^child/, :attributes => {"v" => /m/}})
+ assert !t.match(:descendant => {:attributes => {"v" => false}})
+ end
+
+ def test_match_child_count
+ t = tag("<tag x:k='something'>")
+ tag("hello", t)
+ tag("<child v=john a=kelly>", t)
+ tag("<sib m=vaughn v=james>", t)
+ assert t.match(:children => { :count => 2 })
+ assert t.match(:children => { :count => 2..4 })
+ assert t.match(:children => { :less_than => 4 })
+ assert t.match(:children => { :greater_than => 1 })
+ assert !t.match(:children => { :count => 3 })
+ end
+
+ def test_conditions_as_strings
+ t = tag("<tag x:k='something'>")
+ assert t.match("tag" => "tag")
+ assert t.match("attributes" => { "x:k" => "something" })
+ assert !t.match("tag" => "gat")
+ assert !t.match("attributes" => { "x:j" => "something" })
+ end
+
+ def test_attributes_as_symbols
+ t = tag("<child v=john a=kelly>")
+ assert t.match(:attributes => { :v => /oh/ })
+ assert t.match(:attributes => { :a => /ll/ })
+ end
+
+ def test_match_sibling
+ t = tag("<tag x:k='something'>")
+ tag("hello", t)
+ tag("<span a=b>", t)
+ tag("world", t)
+ m = tag("<span k=r>", t)
+ tag("<span m=l>", t)
+
+ assert m.match(:sibling => {:tag => "span", :attributes => {:a => true}})
+ assert m.match(:sibling => {:tag => "span", :attributes => {:m => true}})
+ assert !m.match(:sibling => {:tag => "span", :attributes => {:k => true}})
+ end
+
+ def test_match_sibling_before
+ t = tag("<tag x:k='something'>")
+ tag("hello", t)
+ tag("<span a=b>", t)
+ tag("world", t)
+ m = tag("<span k=r>", t)
+ tag("<span m=l>", t)
+
+ assert m.match(:before => {:tag => "span", :attributes => {:m => true}})
+ assert !m.match(:before => {:tag => "span", :attributes => {:a => true}})
+ assert !m.match(:before => {:tag => "span", :attributes => {:k => true}})
+ end
+
+ def test_match_sibling_after
+ t = tag("<tag x:k='something'>")
+ tag("hello", t)
+ tag("<span a=b>", t)
+ tag("world", t)
+ m = tag("<span k=r>", t)
+ tag("<span m=l>", t)
+
+ assert m.match(:after => {:tag => "span", :attributes => {:a => true}})
+ assert !m.match(:after => {:tag => "span", :attributes => {:m => true}})
+ assert !m.match(:after => {:tag => "span", :attributes => {:k => true}})
+ end
+
+ def test_to_s
+ t = tag("<b x='foo'>")
+ tag("hello", t)
+ tag("<hr />", t)
+ assert_equal %(<b x="foo">hello<hr /></b>), t.to_s
+ end
+
+ private
+
+ def tag(content, parent=nil)
+ node = HTML::Node.parse(parent,0,0,content)
+ parent.children << node if parent
+ node
+ end
+end
diff --git a/actionpack/test/controller/html-scanner/text_node_test.rb b/actionpack/test/controller/html-scanner/text_node_test.rb
new file mode 100644
index 0000000000..9853701fda
--- /dev/null
+++ b/actionpack/test/controller/html-scanner/text_node_test.rb
@@ -0,0 +1,51 @@
+require File.dirname(__FILE__) + '/../../abstract_unit'
+require 'test/unit'
+
+class TextNodeTest < Test::Unit::TestCase
+ def setup
+ @node = HTML::Text.new(nil, 0, 0, "hello, howdy, aloha, annyeong")
+ end
+
+ def test_to_s
+ assert_equal "hello, howdy, aloha, annyeong", @node.to_s
+ end
+
+ def test_find_string
+ assert_equal @node, @node.find("hello, howdy, aloha, annyeong")
+ assert_equal false, @node.find("bogus")
+ end
+
+ def test_find_regexp
+ assert_equal @node, @node.find(/an+y/)
+ assert_nil @node.find(/b/)
+ end
+
+ def test_find_hash
+ assert_equal @node, @node.find(:content => /howdy/)
+ assert_nil @node.find(:content => /^howdy$/)
+ assert_equal false, @node.find(:content => "howdy")
+ end
+
+ def test_find_other
+ assert_nil @node.find(:hello)
+ end
+
+ def test_match_string
+ assert @node.match("hello, howdy, aloha, annyeong")
+ assert_equal false, @node.match("bogus")
+ end
+
+ def test_match_regexp
+ assert_not_nil @node, @node.match(/an+y/)
+ assert_nil @node.match(/b/)
+ end
+
+ def test_match_hash
+ assert_not_nil @node, @node.match(:content => "howdy")
+ assert_nil @node.match(:content => /^howdy$/)
+ end
+
+ def test_match_other
+ assert_nil @node.match(:hello)
+ end
+end \ No newline at end of file
diff --git a/actionpack/test/controller/html-scanner/tokenizer_test.rb b/actionpack/test/controller/html-scanner/tokenizer_test.rb
new file mode 100644
index 0000000000..437136b9a7
--- /dev/null
+++ b/actionpack/test/controller/html-scanner/tokenizer_test.rb
@@ -0,0 +1,125 @@
+require File.dirname(__FILE__) + '/../../abstract_unit'
+require 'test/unit'
+
+class TokenizerTest < Test::Unit::TestCase
+
+ def test_blank
+ tokenize ""
+ assert_end
+ end
+
+ def test_space
+ tokenize " "
+ assert_next " "
+ assert_end
+ end
+
+ def test_tag_simple_open
+ tokenize "<tag>"
+ assert_next "<tag>"
+ assert_end
+ end
+
+ def test_tag_simple_self_closing
+ tokenize "<tag />"
+ assert_next "<tag />"
+ assert_end
+ end
+
+ def test_tag_simple_closing
+ tokenize "</tag>"
+ assert_next "</tag>"
+ end
+
+ def test_tag_with_single_quoted_attribute
+ tokenize %{<tag a='hello'>x}
+ assert_next %{<tag a='hello'>}
+ end
+
+ def test_tag_with_single_quoted_attribute_with_escape
+ tokenize %{<tag a='hello\\''>x}
+ assert_next %{<tag a='hello\\''>}
+ end
+
+ def test_tag_with_double_quoted_attribute
+ tokenize %{<tag a="hello">x}
+ assert_next %{<tag a="hello">}
+ end
+
+ def test_tag_with_double_quoted_attribute_with_escape
+ tokenize %{<tag a="hello\\"">x}
+ assert_next %{<tag a="hello\\"">}
+ end
+
+ def test_tag_with_unquoted_attribute
+ tokenize %{<tag a=hello>x}
+ assert_next %{<tag a=hello>}
+ end
+
+ def test_tag_with_lt_char_in_attribute
+ tokenize %{<tag a="x < y">x}
+ assert_next %{<tag a="x < y">}
+ end
+
+ def test_tag_with_gt_char_in_attribute
+ tokenize %{<tag a="x > y">x}
+ assert_next %{<tag a="x > y">}
+ end
+
+ def test_doctype_tag
+ tokenize %{<!DOCTYPE "blah" "blah" "blah">\n <html>}
+ assert_next %{<!DOCTYPE "blah" "blah" "blah">}
+ assert_next %{\n }
+ assert_next %{<html>}
+ end
+
+ def test_cdata_tag
+ tokenize %{<![CDATA[<br>]]>}
+ assert_next %{<![CDATA[<br>]]>}
+ assert_end
+ end
+
+ def test_less_than_with_space
+ tokenize %{original < hello > world}
+ assert_next %{original }
+ assert_next %{< hello > world}
+ end
+
+ def test_less_than_without_matching_greater_than
+ tokenize %{hello <span onmouseover="gotcha"\n<b>foo</b>\nbar</span>}
+ assert_next %{hello }
+ assert_next %{<span onmouseover="gotcha"\n}
+ assert_next %{<b>}
+ assert_next %{foo}
+ assert_next %{</b>}
+ assert_next %{\nbar}
+ assert_next %{</span>}
+ assert_end
+ end
+
+ def test_unterminated_comment
+ tokenize %{hello <!-- neverending...}
+ assert_next %{hello }
+ assert_next %{<!-- neverending...}
+ assert_end
+ end
+
+ private
+
+ def tokenize(text)
+ @tokenizer = HTML::Tokenizer.new(text)
+ end
+
+ def assert_next(expected, message=nil)
+ token = @tokenizer.next
+ assert_equal expected, token, message
+ end
+
+ def assert_sequence(*expected)
+ assert_next expected.shift until expected.empty?
+ end
+
+ def assert_end(message=nil)
+ assert_nil @tokenizer.next, message
+ end
+end