diff options
Diffstat (limited to 'guides/rails_guides')
-rw-r--r-- | guides/rails_guides/generator.rb | 249 | ||||
-rw-r--r-- | guides/rails_guides/helpers.rb | 53 | ||||
-rw-r--r-- | guides/rails_guides/indexer.rb | 68 | ||||
-rw-r--r-- | guides/rails_guides/kindle.rb | 119 | ||||
-rw-r--r-- | guides/rails_guides/levenshtein.rb | 40 | ||||
-rw-r--r-- | guides/rails_guides/markdown.rb | 165 | ||||
-rw-r--r-- | guides/rails_guides/markdown/renderer.rb | 83 |
7 files changed, 777 insertions, 0 deletions
diff --git a/guides/rails_guides/generator.rb b/guides/rails_guides/generator.rb new file mode 100644 index 0000000000..7618fce2c8 --- /dev/null +++ b/guides/rails_guides/generator.rb @@ -0,0 +1,249 @@ +# --------------------------------------------------------------------------- +# +# This script generates the guides. It can be invoked via the +# guides:generate rake task within the guides directory. +# +# Guides are taken from the source directory, and the resulting HTML goes into the +# output directory. Assets are stored under files, and copied to output/files as +# part of the generation process. +# +# Some arguments may be passed via environment variables: +# +# WARNINGS +# If you are writing a guide, please work always with WARNINGS=1. Users can +# generate the guides, and thus this flag is off by default. +# +# Internal links (anchors) are checked. If a reference is broken levenshtein +# distance is used to suggest an existing one. This is useful since IDs are +# generated by Markdown from headers and thus edits alter them. +# +# Also detects duplicated IDs. They happen if there are headers with the same +# text. Please do resolve them, if any, so guides are valid XHTML. +# +# ALL +# Set to "1" to force the generation of all guides. +# +# ONLY +# Use ONLY if you want to generate only one or a set of guides. Prefixes are +# enough: +# +# # generates only association_basics.html +# ONLY=assoc ruby rails_guides.rb +# +# Separate many using commas: +# +# # generates only association_basics.html and migrations.html +# ONLY=assoc,migrations ruby rails_guides.rb +# +# Note that if you are working on a guide generation will by default process +# only that one, so ONLY is rarely used nowadays. +# +# GUIDES_LANGUAGE +# Use GUIDES_LANGUAGE when you want to generate translated guides in +# <tt>source/<GUIDES_LANGUAGE></tt> folder (such as <tt>source/es</tt>). +# Ignore it when generating English guides. +# +# EDGE +# Set to "1" to indicate generated guides should be marked as edge. This +# inserts a badge and changes the preamble of the home page. +# +# --------------------------------------------------------------------------- + +require 'set' +require 'fileutils' + +require 'active_support/core_ext/string/output_safety' +require 'active_support/core_ext/object/blank' +require 'action_controller' +require 'action_view' + +require 'rails_guides/markdown' +require 'rails_guides/indexer' +require 'rails_guides/helpers' +require 'rails_guides/levenshtein' + +module RailsGuides + class Generator + attr_reader :guides_dir, :source_dir, :output_dir, :edge, :warnings, :all + + GUIDES_RE = /\.(?:erb|md)\z/ + + def initialize(output=nil) + set_flags_from_environment + + if kindle? + check_for_kindlegen + register_kindle_mime_types + end + + initialize_dirs(output) + create_output_dir_if_needed + end + + def set_flags_from_environment + @edge = ENV['EDGE'] == '1' + @warnings = ENV['WARNINGS'] == '1' + @all = ENV['ALL'] == '1' + @kindle = ENV['KINDLE'] == '1' + @version = ENV['RAILS_VERSION'] || 'local' + @lang = ENV['GUIDES_LANGUAGE'] + end + + def register_kindle_mime_types + Mime::Type.register_alias("application/xml", :opf, %w(opf)) + Mime::Type.register_alias("application/xml", :ncx, %w(ncx)) + end + + def generate + generate_guides + copy_assets + generate_mobi if kindle? + end + + private + + def kindle? + @kindle + end + + def check_for_kindlegen + if `which kindlegen`.blank? + raise "Can't create a kindle version without `kindlegen`." + end + end + + def generate_mobi + require 'rails_guides/kindle' + out = "#{output_dir}/kindlegen.out" + Kindle.generate(output_dir, mobi, out) + puts "(kindlegen log at #{out})." + end + + def mobi + "ruby_on_rails_guides_#@version%s.mobi" % (@lang.present? ? ".#@lang" : '') + end + + def initialize_dirs(output) + @guides_dir = File.join(File.dirname(__FILE__), '..') + @source_dir = "#@guides_dir/source/#@lang" + @output_dir = if output + output + elsif kindle? + "#@guides_dir/output/kindle/#@lang" + else + "#@guides_dir/output/#@lang" + end.sub(%r</$>, '') + end + + def create_output_dir_if_needed + FileUtils.mkdir_p(output_dir) + end + + def generate_guides + guides_to_generate.each do |guide| + output_file = output_file_for(guide) + generate_guide(guide, output_file) if generate?(guide, output_file) + end + end + + def guides_to_generate + guides = Dir.entries(source_dir).grep(GUIDES_RE) + + if kindle? + Dir.entries("#{source_dir}/kindle").grep(GUIDES_RE).map do |entry| + next if entry == 'KINDLE.md' + guides << "kindle/#{entry}" + end + end + + ENV.key?('ONLY') ? select_only(guides) : guides + end + + def select_only(guides) + prefixes = ENV['ONLY'].split(",").map(&:strip) + guides.select do |guide| + guide.start_with?('kindle'.freeze, *prefixes) + end + end + + def copy_assets + FileUtils.cp_r(Dir.glob("#{guides_dir}/assets/*"), output_dir) + end + + def output_file_for(guide) + if guide.end_with?('.md') + guide.sub(/md\z/, 'html') + else + guide.sub(/\.erb\z/, '') + end + end + + def output_path_for(output_file) + File.join(output_dir, File.basename(output_file)) + end + + def generate?(source_file, output_file) + fin = File.join(source_dir, source_file) + fout = output_path_for(output_file) + all || !File.exist?(fout) || File.mtime(fout) < File.mtime(fin) + end + + def generate_guide(guide, output_file) + output_path = output_path_for(output_file) + puts "Generating #{guide} as #{output_file}" + layout = kindle? ? 'kindle/layout' : 'layout' + + File.open(output_path, 'w') do |f| + view = ActionView::Base.new(source_dir, :edge => @edge, :version => @version, :mobi => "kindle/#{mobi}", :lang => @lang) + view.extend(Helpers) + + if guide =~ /\.(\w+)\.erb$/ + # Generate the special pages like the home. + # Passing a template handler in the template name is deprecated. So pass the file name without the extension. + result = view.render(:layout => layout, :formats => [$1], :file => $`) + else + body = File.read(File.join(source_dir, guide)) + result = RailsGuides::Markdown.new(view, layout).render(body) + + warn_about_broken_links(result) if @warnings + end + + f.write(result) + end + end + + def warn_about_broken_links(html) + anchors = extract_anchors(html) + check_fragment_identifiers(html, anchors) + end + + def extract_anchors(html) + # Markdown generates headers with IDs computed from titles. + anchors = Set.new + html.scan(/<h\d\s+id="([^"]+)/).flatten.each do |anchor| + if anchors.member?(anchor) + puts "*** DUPLICATE ID: #{anchor}, please make sure that there're no headings with the same name at the same level." + else + anchors << anchor + end + end + + # Footnotes. + anchors += Set.new(html.scan(/<p\s+class="footnote"\s+id="([^"]+)/).flatten) + anchors += Set.new(html.scan(/<sup\s+class="footnote"\s+id="([^"]+)/).flatten) + return anchors + end + + def check_fragment_identifiers(html, anchors) + html.scan(/<a\s+href="#([^"]+)/).flatten.each do |fragment_identifier| + next if fragment_identifier == 'mainCol' # in layout, jumps to some DIV + unless anchors.member?(fragment_identifier) + guess = anchors.min { |a, b| + Levenshtein.distance(fragment_identifier, a) <=> Levenshtein.distance(fragment_identifier, b) + } + puts "*** BROKEN LINK: ##{fragment_identifier}, perhaps you meant ##{guess}." + end + end + end + end +end diff --git a/guides/rails_guides/helpers.rb b/guides/rails_guides/helpers.rb new file mode 100644 index 0000000000..5bf73da16c --- /dev/null +++ b/guides/rails_guides/helpers.rb @@ -0,0 +1,53 @@ +require 'yaml' + +module RailsGuides + module Helpers + def guide(name, url, options = {}, &block) + link = content_tag(:a, :href => url) { name } + result = content_tag(:dt, link) + + if options[:work_in_progress] + result << content_tag(:dd, 'Work in progress', :class => 'work-in-progress') + end + + result << content_tag(:dd, capture(&block)) + result + end + + def documents_by_section + @documents_by_section ||= YAML.load_file(File.expand_path("../../source/#{@lang ? @lang + '/' : ''}documents.yaml", __FILE__)) + end + + def documents_flat + documents_by_section.flat_map {|section| section['documents']} + end + + def finished_documents(documents) + documents.reject { |document| document['work_in_progress'] } + end + + def docs_for_menu(position=nil) + if position.nil? + documents_by_section + elsif position == 'L' + documents_by_section.to(3) + else + documents_by_section.from(4) + end + end + + def author(name, nick, image = 'credits_pic_blank.gif', &block) + image = "images/#{image}" + + result = tag(:img, :src => image, :class => 'left pic', :alt => name, :width => 91, :height => 91) + result << content_tag(:h3, name) + result << content_tag(:p, capture(&block)) + content_tag(:div, result, :class => 'clearfix', :id => nick) + end + + def code(&block) + c = capture(&block) + content_tag(:code, c) + end + end +end diff --git a/guides/rails_guides/indexer.rb b/guides/rails_guides/indexer.rb new file mode 100644 index 0000000000..89fbccbb1d --- /dev/null +++ b/guides/rails_guides/indexer.rb @@ -0,0 +1,68 @@ +require 'active_support/core_ext/object/blank' +require 'active_support/core_ext/string/inflections' + +module RailsGuides + class Indexer + attr_reader :body, :result, :warnings, :level_hash + + def initialize(body, warnings) + @body = body + @result = @body.dup + @warnings = warnings + end + + def index + @level_hash = process(body) + end + + private + + def process(string, current_level=3, counters=[1]) + s = StringScanner.new(string) + + level_hash = {} + + while !s.eos? + re = %r{^h(\d)(?:\((#.*?)\))?\s*\.\s*(.*)$} + s.match?(re) + if matched = s.matched + matched =~ re + level, idx, title = $1.to_i, $2, $3.strip + + if level < current_level + # This is needed. Go figure. + return level_hash + elsif level == current_level + index = counters.join(".") + idx ||= '#' + title_to_idx(title) + + raise "Parsing Fail" unless @result.sub!(matched, "h#{level}(#{idx}). #{index} #{title}") + + key = { + :title => title, + :id => idx + } + # Recurse + counters << 1 + level_hash[key] = process(s.post_match, current_level + 1, counters) + counters.pop + + # Increment the current level + last = counters.pop + counters << last + 1 + end + end + s.getch + end + level_hash + end + + def title_to_idx(title) + idx = title.strip.parameterize.sub(/^\d+/, '') + if warnings && idx.blank? + puts "BLANK ID: please put an explicit ID for section #{title}, as in h5(#my-id)" + end + idx + end + end +end diff --git a/guides/rails_guides/kindle.rb b/guides/rails_guides/kindle.rb new file mode 100644 index 0000000000..081afcb09f --- /dev/null +++ b/guides/rails_guides/kindle.rb @@ -0,0 +1,119 @@ +#!/usr/bin/env ruby + +unless `which kindlerb` + abort "Please gem install kindlerb" +end + +require 'nokogiri' +require 'fileutils' +require 'yaml' +require 'date' + +module Kindle + extend self + + def generate(output_dir, mobi_outfile, logfile) + output_dir = File.absolute_path(output_dir) + Dir.chdir output_dir do + puts "=> Using output dir: #{output_dir}" + puts "=> Arranging html pages in document order" + toc = File.read("toc.ncx") + doc = Nokogiri::XML(toc).xpath("//ncx:content", 'ncx' => "http://www.daisy.org/z3986/2005/ncx/") + html_pages = doc.select {|c| c[:src]}.map {|c| c[:src]}.uniq + + generate_front_matter(html_pages) + + generate_sections(html_pages) + + generate_document_metadata(mobi_outfile) + + puts "Creating MOBI document with kindlegen. This may take a while." + cmd = "kindlerb . > #{File.absolute_path logfile} 2>&1" + puts cmd + system(cmd) + puts "MOBI document generated at #{File.expand_path(mobi_outfile, output_dir)}" + end + end + + def generate_front_matter(html_pages) + frontmatter = [] + html_pages.delete_if {|x| + if x =~ /(toc|welcome|credits|copyright).html/ + frontmatter << x unless x =~ /toc/ + true + end + } + html = frontmatter.map {|x| + Nokogiri::HTML(File.open(x)).at("body").inner_html + }.join("\n") + + fdoc = Nokogiri::HTML(html) + fdoc.search("h3").each do |h3| + h3.name = 'h4' + end + fdoc.search("h2").each do |h2| + h2.name = 'h3' + h2['id'] = h2.inner_text.gsub(/\s/, '-') + end + add_head_section fdoc, "Front Matter" + File.open("frontmatter.html",'w') {|f| f.puts fdoc.to_html} + html_pages.unshift "frontmatter.html" + end + + def generate_sections(html_pages) + FileUtils::rm_rf("sections/") + html_pages.each_with_index do |page, section_idx| + FileUtils::mkdir_p("sections/%03d" % section_idx) + doc = Nokogiri::HTML(File.open(page)) + title = doc.at("title").inner_text.gsub("Ruby on Rails Guides: ", '') + title = page.capitalize.gsub('.html', '') if title.strip == '' + File.open("sections/%03d/_section.txt" % section_idx, 'w') {|f| f.puts title} + doc.xpath("//h3[@id]").each_with_index do |h3,item_idx| + subsection = h3.inner_text + content = h3.xpath("./following-sibling::*").take_while {|x| x.name != "h3"}.map(&:to_html) + item = Nokogiri::HTML(h3.to_html + content.join("\n")) + item_path = "sections/%03d/%03d.html" % [section_idx, item_idx] + add_head_section(item, subsection) + item.search("img").each do |img| + img['src'] = "#{Dir.pwd}/#{img['src']}" + end + item.xpath("//li/p").each {|p| p.swap(p.children); p.remove} + File.open(item_path, 'w') {|f| f.puts item.to_html} + end + end + end + + def generate_document_metadata(mobi_outfile) + puts "=> Generating _document.yml" + x = Nokogiri::XML(File.open("rails_guides.opf")).remove_namespaces! + cover_jpg = "#{Dir.pwd}/images/rails_guides_kindle_cover.jpg" + cover_gif = cover_jpg.sub(/jpg$/, 'gif') + puts `convert #{cover_jpg} #{cover_gif}` + document = { + 'doc_uuid' => x.at("package")['unique-identifier'], + 'title' => x.at("title").inner_text.gsub(/\(.*$/, " v2"), + 'publisher' => x.at("publisher").inner_text, + 'author' => x.at("creator").inner_text, + 'subject' => x.at("subject").inner_text, + 'date' => x.at("date").inner_text, + 'cover' => cover_gif, + 'masthead' => nil, + 'mobi_outfile' => mobi_outfile + } + puts document.to_yaml + File.open("_document.yml", 'w'){|f| f.puts document.to_yaml} + end + + def add_head_section(doc, title) + head = Nokogiri::XML::Node.new "head", doc + title_node = Nokogiri::XML::Node.new "title", doc + title_node.content = title + title_node.parent = head + css = Nokogiri::XML::Node.new "link", doc + css['rel'] = 'stylesheet' + css['type'] = 'text/css' + css['href'] = "#{Dir.pwd}/stylesheets/kindle.css" + css.parent = head + doc.at("body").before head + end +end diff --git a/guides/rails_guides/levenshtein.rb b/guides/rails_guides/levenshtein.rb new file mode 100644 index 0000000000..049f633258 --- /dev/null +++ b/guides/rails_guides/levenshtein.rb @@ -0,0 +1,40 @@ +module RailsGuides + module Levenshtein + # This code is based directly on the Text gem implementation + # Returns a value representing the "cost" of transforming str1 into str2 + def self.distance str1, str2 + s = str1 + t = str2 + n = s.length + m = t.length + + return m if (0 == n) + return n if (0 == m) + + d = (0..m).to_a + x = nil + + # avoid duplicating an enumerable object in the loop + str2_codepoint_enumerable = str2.each_codepoint + + str1.each_codepoint.with_index do |char1, i| + e = i+1 + + str2_codepoint_enumerable.with_index do |char2, j| + cost = (char1 == char2) ? 0 : 1 + x = [ + d[j+1] + 1, # insertion + e + 1, # deletion + d[j] + cost # substitution + ].min + d[j] = e + e = x + end + + d[m] = x + end + + return x + end + end +end diff --git a/guides/rails_guides/markdown.rb b/guides/rails_guides/markdown.rb new file mode 100644 index 0000000000..69c7cd5136 --- /dev/null +++ b/guides/rails_guides/markdown.rb @@ -0,0 +1,165 @@ +require 'redcarpet' +require 'nokogiri' +require 'rails_guides/markdown/renderer' + +module RailsGuides + class Markdown + def initialize(view, layout) + @view = view + @layout = layout + @index_counter = Hash.new(0) + @raw_header = '' + @node_ids = {} + end + + def render(body) + @raw_body = body + extract_raw_header_and_body + generate_header + generate_title + generate_body + generate_structure + generate_index + render_page + end + + private + + def dom_id(nodes) + dom_id = dom_id_text(nodes.last.text) + + # Fix duplicate node by prefix with its parent node + if @node_ids[dom_id] + if @node_ids[dom_id].size > 1 + duplicate_nodes = @node_ids.delete(dom_id) + new_node_id = "#{duplicate_nodes[-2][:id]}-#{duplicate_nodes.last[:id]}" + duplicate_nodes.last[:id] = new_node_id + @node_ids[new_node_id] = duplicate_nodes + end + + dom_id = "#{nodes[-2][:id]}-#{dom_id}" + end + + @node_ids[dom_id] = nodes + dom_id + end + + def dom_id_text(text) + escaped_chars = Regexp.escape('\\/`*_{}[]()#+-.!:,;|&<>^~=\'"') + + text.downcase.gsub(/\?/, '-questionmark') + .gsub(/!/, '-bang') + .gsub(/[#{escaped_chars}]+/, ' ').strip + .gsub(/\s+/, '-') + end + + def engine + @engine ||= Redcarpet::Markdown.new(Renderer, { + no_intra_emphasis: true, + fenced_code_blocks: true, + autolink: true, + strikethrough: true, + superscript: true, + tables: true + }) + end + + def extract_raw_header_and_body + if @raw_body =~ /^\-{40,}$/ + @raw_header, _, @raw_body = @raw_body.partition(/^\-{40,}$/).map(&:strip) + end + end + + def generate_body + @body = engine.render(@raw_body) + end + + def generate_header + @header = engine.render(@raw_header).html_safe + end + + def generate_structure + @headings_for_index = [] + if @body.present? + @body = Nokogiri::HTML.fragment(@body).tap do |doc| + hierarchy = [] + + doc.children.each do |node| + if node.name =~ /^h[3-6]$/ + case node.name + when 'h3' + hierarchy = [node] + @headings_for_index << [1, node, node.inner_html] + when 'h4' + hierarchy = hierarchy[0, 1] + [node] + @headings_for_index << [2, node, node.inner_html] + when 'h5' + hierarchy = hierarchy[0, 2] + [node] + when 'h6' + hierarchy = hierarchy[0, 3] + [node] + end + + node[:id] = dom_id(hierarchy) + node.inner_html = "#{node_index(hierarchy)} #{node.inner_html}" + end + end + end.to_html + end + end + + def generate_index + if @headings_for_index.present? + raw_index = '' + @headings_for_index.each do |level, node, label| + if level == 1 + raw_index += "1. [#{label}](##{node[:id]})\n" + elsif level == 2 + raw_index += " * [#{label}](##{node[:id]})\n" + end + end + + @index = Nokogiri::HTML.fragment(engine.render(raw_index)).tap do |doc| + doc.at('ol')[:class] = 'chapters' + end.to_html + + @index = <<-INDEX.html_safe + <div id="subCol"> + <h3 class="chapter"><img src="images/chapters_icon.gif" alt="" />Chapters</h3> + #{@index} + </div> + INDEX + end + end + + def generate_title + if heading = Nokogiri::HTML.fragment(@header).at(:h2) + @title = "#{heading.text} — Ruby on Rails Guides" + else + @title = "Ruby on Rails Guides" + end + end + + def node_index(hierarchy) + case hierarchy.size + when 1 + @index_counter[2] = @index_counter[3] = @index_counter[4] = 0 + "#{@index_counter[1] += 1}" + when 2 + @index_counter[3] = @index_counter[4] = 0 + "#{@index_counter[1]}.#{@index_counter[2] += 1}" + when 3 + @index_counter[4] = 0 + "#{@index_counter[1]}.#{@index_counter[2]}.#{@index_counter[3] += 1}" + when 4 + "#{@index_counter[1]}.#{@index_counter[2]}.#{@index_counter[3]}.#{@index_counter[4] += 1}" + end + end + + def render_page + @view.content_for(:header_section) { @header } + @view.content_for(:page_title) { @title } + @view.content_for(:index_section) { @index } + @view.render(:layout => @layout, :text => @body) + end + end +end diff --git a/guides/rails_guides/markdown/renderer.rb b/guides/rails_guides/markdown/renderer.rb new file mode 100644 index 0000000000..554d94ad50 --- /dev/null +++ b/guides/rails_guides/markdown/renderer.rb @@ -0,0 +1,83 @@ +module RailsGuides + class Markdown + class Renderer < Redcarpet::Render::HTML + def initialize(options={}) + super + end + + def block_code(code, language) + <<-HTML +<div class="code_container"> +<pre class="brush: #{brush_for(language)}; gutter: false; toolbar: false"> +#{ERB::Util.h(code)} +</pre> +</div> +HTML + end + + def header(text, header_level) + # Always increase the heading level by, so we can use h1, h2 heading in the document + header_level += 1 + + %(<h#{header_level}>#{text}</h#{header_level}>) + end + + def paragraph(text) + if text =~ /^(TIP|IMPORTANT|CAUTION|WARNING|NOTE|INFO|TODO)[.:]/ + convert_notes(text) + elsif text.include?('DO NOT READ THIS FILE ON GITHUB') + elsif text =~ /^\[<sup>(\d+)\]:<\/sup> (.+)$/ + linkback = %(<a href="#footnote-#{$1}-ref"><sup>#{$1}</sup></a>) + %(<p class="footnote" id="footnote-#{$1}">#{linkback} #{$2}</p>) + else + text = convert_footnotes(text) + "<p>#{text}</p>" + end + end + + private + + def convert_footnotes(text) + text.gsub(/\[<sup>(\d+)\]<\/sup>/i) do + %(<sup class="footnote" id="footnote-#{$1}-ref">) + + %(<a href="#footnote-#{$1}">#{$1}</a></sup>) + end + end + + def brush_for(code_type) + case code_type + when 'ruby', 'sql', 'plain' + code_type + when 'erb', 'html+erb' + 'ruby; html-script: true' + when 'html' + 'xml' # HTML is understood, but there are .xml rules in the CSS + else + 'plain' + end + end + + def convert_notes(body) + # The following regexp detects special labels followed by a + # paragraph, perhaps at the end of the document. + # + # It is important that we do not eat more than one newline + # because formatting may be wrong otherwise. For example, + # if a bulleted list follows the first item is not rendered + # as a list item, but as a paragraph starting with a plain + # asterisk. + body.gsub(/^(TIP|IMPORTANT|CAUTION|WARNING|NOTE|INFO|TODO)[.:](.*?)(\n(?=\n)|\Z)/m) do + css_class = case $1 + when 'CAUTION', 'IMPORTANT' + 'warning' + when 'TIP' + 'info' + else + $1.downcase + end + %(<div class="#{css_class}"><p>#{$2.strip}</p></div>) + end + end + end + end +end |