diff options
Diffstat (limited to 'guides/rails_guides')
-rw-r--r-- | guides/rails_guides/generator.rb | 219 | ||||
-rw-r--r-- | guides/rails_guides/helpers.rb | 46 | ||||
-rw-r--r-- | guides/rails_guides/indexer.rb | 70 | ||||
-rw-r--r-- | guides/rails_guides/kindle.rb | 116 | ||||
-rw-r--r-- | guides/rails_guides/levenshtein.rb | 44 | ||||
-rw-r--r-- | guides/rails_guides/markdown.rb | 173 | ||||
-rw-r--r-- | guides/rails_guides/markdown/renderer.rb | 128 |
7 files changed, 796 insertions, 0 deletions
diff --git a/guides/rails_guides/generator.rb b/guides/rails_guides/generator.rb new file mode 100644 index 0000000000..48e90510e1 --- /dev/null +++ b/guides/rails_guides/generator.rb @@ -0,0 +1,219 @@ +# frozen_string_literal: true + +require "set" +require "fileutils" + +require "active_support/core_ext/string/output_safety" +require "active_support/core_ext/object/blank" +require "action_controller" +require "action_view" + +require "rails_guides/markdown" +require "rails_guides/indexer" +require "rails_guides/helpers" +require "rails_guides/levenshtein" + +module RailsGuides + class Generator + GUIDES_RE = /\.(?:erb|md)\z/ + + def initialize(edge:, version:, all:, only:, kindle:, language:, direction: "ltr") + @edge = edge + @version = version + @all = all + @only = only + @kindle = kindle + @language = language + @direction = direction + + if @kindle + check_for_kindlegen + register_kindle_mime_types + end + + initialize_dirs + create_output_dir_if_needed + initialize_markdown_renderer + end + + def generate + generate_guides + copy_assets + generate_mobi if @kindle + end + + private + + def register_kindle_mime_types + Mime::Type.register_alias("application/xml", :opf, %w(opf)) + Mime::Type.register_alias("application/xml", :ncx, %w(ncx)) + end + + def check_for_kindlegen + if `which kindlegen`.blank? + raise "Can't create a kindle version without `kindlegen`." + end + end + + def generate_mobi + require "rails_guides/kindle" + out = "#{@output_dir}/kindlegen.out" + Kindle.generate(@output_dir, mobi, out) + puts "(kindlegen log at #{out})." + end + + def mobi + mobi = "ruby_on_rails_guides_#{@version || @edge[0, 7]}" + mobi += ".#{@language}" if @language + mobi += ".mobi" + end + + def initialize_dirs + @guides_dir = File.expand_path("..", __dir__) + + @source_dir = "#{@guides_dir}/source" + @source_dir += "/#{@language}" if @language + + @output_dir = "#{@guides_dir}/output" + @output_dir += "/kindle" if @kindle + @output_dir += "/#{@language}" if @language + end + + def create_output_dir_if_needed + FileUtils.mkdir_p(@output_dir) + end + + def initialize_markdown_renderer + Markdown::Renderer.edge = @edge + Markdown::Renderer.version = @version + end + + def generate_guides + guides_to_generate.each do |guide| + output_file = output_file_for(guide) + generate_guide(guide, output_file) if generate?(guide, output_file) + end + end + + def guides_to_generate + guides = Dir.entries(@source_dir).grep(GUIDES_RE) + + if @kindle + Dir.entries("#{@source_dir}/kindle").grep(GUIDES_RE).map do |entry| + next if entry == "KINDLE.md" + guides << "kindle/#{entry}" + end + end + + @only ? select_only(guides) : guides + end + + def select_only(guides) + prefixes = @only.split(",").map(&:strip) + guides.select do |guide| + guide.start_with?("kindle", *prefixes) + end + end + + def copy_assets + FileUtils.cp_r(Dir.glob("#{@guides_dir}/assets/*"), @output_dir) + + if @direction == "rtl" + overwrite_css_with_right_to_left_direction + end + end + + def overwrite_css_with_right_to_left_direction + FileUtils.mv("#{@output_dir}/stylesheets/main.rtl.css", "#{@output_dir}/stylesheets/main.css") + end + + def output_file_for(guide) + if guide.end_with?(".md") + guide.sub(/md\z/, "html") + else + guide.sub(/\.erb\z/, "") + end + end + + def output_path_for(output_file) + File.join(@output_dir, File.basename(output_file)) + end + + def generate?(source_file, output_file) + fin = File.join(@source_dir, source_file) + fout = output_path_for(output_file) + @all || !File.exist?(fout) || File.mtime(fout) < File.mtime(fin) + end + + def generate_guide(guide, output_file) + output_path = output_path_for(output_file) + puts "Generating #{guide} as #{output_file}" + layout = @kindle ? "kindle/layout" : "layout" + + view = ActionView::Base.new( + @source_dir, + edge: @edge, + version: @version, + mobi: "kindle/#{mobi}", + language: @language + ) + view.extend(Helpers) + + if guide =~ /\.(\w+)\.erb$/ + return if %w[_license _welcome layout].include?($`) + + # Generate the special pages like the home. + # Passing a template handler in the template name is deprecated. So pass the file name without the extension. + result = view.render(layout: layout, formats: [$1], file: $`) + else + body = File.read("#{@source_dir}/#{guide}") + result = RailsGuides::Markdown.new( + view: view, + layout: layout, + edge: @edge, + version: @version + ).render(body) + + warn_about_broken_links(result) + end + + File.open(output_path, "w") do |f| + f.write(result) + end + end + + def warn_about_broken_links(html) + anchors = extract_anchors(html) + check_fragment_identifiers(html, anchors) + end + + def extract_anchors(html) + # Markdown generates headers with IDs computed from titles. + anchors = Set.new + html.scan(/<h\d\s+id="([^"]+)/).flatten.each do |anchor| + if anchors.member?(anchor) + puts "*** DUPLICATE ID: #{anchor}, please make sure that there're no headings with the same name at the same level." + else + anchors << anchor + end + end + + # Footnotes. + anchors += Set.new(html.scan(/<p\s+class="footnote"\s+id="([^"]+)/).flatten) + anchors += Set.new(html.scan(/<sup\s+class="footnote"\s+id="([^"]+)/).flatten) + anchors + end + + def check_fragment_identifiers(html, anchors) + html.scan(/<a\s+href="#([^"]+)/).flatten.each do |fragment_identifier| + next if fragment_identifier == "mainCol" # in layout, jumps to some DIV + unless anchors.member?(CGI.unescape(fragment_identifier)) + guess = anchors.min { |a, b| + Levenshtein.distance(fragment_identifier, a) <=> Levenshtein.distance(fragment_identifier, b) + } + puts "*** BROKEN LINK: ##{fragment_identifier}, perhaps you meant ##{guess}." + end + end + end + end +end diff --git a/guides/rails_guides/helpers.rb b/guides/rails_guides/helpers.rb new file mode 100644 index 0000000000..5ab1388c29 --- /dev/null +++ b/guides/rails_guides/helpers.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +require "yaml" + +module RailsGuides + module Helpers + def guide(name, url, options = {}, &block) + link = content_tag(:a, href: url) { name } + result = content_tag(:dt, link) + + if options[:work_in_progress] + result << content_tag(:dd, "Work in progress", class: "work-in-progress") + end + + result << content_tag(:dd, capture(&block)) + result + end + + def documents_by_section + @documents_by_section ||= YAML.load_file(File.expand_path("../source/#{@language ? @language + '/' : ''}documents.yaml", __dir__)) + end + + def documents_flat + documents_by_section.flat_map { |section| section["documents"] } + end + + def finished_documents(documents) + documents.reject { |document| document["work_in_progress"] } + end + + def docs_for_menu(position = nil) + if position.nil? + documents_by_section + elsif position == "L" + documents_by_section.to(3) + else + documents_by_section.from(4) + end + end + + def code(&block) + c = capture(&block) + content_tag(:code, c) + end + end +end diff --git a/guides/rails_guides/indexer.rb b/guides/rails_guides/indexer.rb new file mode 100644 index 0000000000..c707464cdf --- /dev/null +++ b/guides/rails_guides/indexer.rb @@ -0,0 +1,70 @@ +# frozen_string_literal: true + +require "active_support/core_ext/object/blank" +require "active_support/core_ext/string/inflections" + +module RailsGuides + class Indexer + attr_reader :body, :result, :warnings, :level_hash + + def initialize(body, warnings) + @body = body + @result = @body.dup + @warnings = warnings + end + + def index + @level_hash = process(body) + end + + private + + def process(string, current_level = 3, counters = [1]) + s = StringScanner.new(string) + + level_hash = {} + + while !s.eos? + re = %r{^h(\d)(?:\((#.*?)\))?\s*\.\s*(.*)$} + s.match?(re) + if matched = s.matched + matched =~ re + level, idx, title = $1.to_i, $2, $3.strip + + if level < current_level + # This is needed. Go figure. + return level_hash + elsif level == current_level + index = counters.join(".") + idx ||= "#" + title_to_idx(title) + + raise "Parsing Fail" unless @result.sub!(matched, "h#{level}(#{idx}). #{index} #{title}") + + key = { + title: title, + id: idx + } + # Recurse + counters << 1 + level_hash[key] = process(s.post_match, current_level + 1, counters) + counters.pop + + # Increment the current level + last = counters.pop + counters << last + 1 + end + end + s.getch + end + level_hash + end + + def title_to_idx(title) + idx = title.strip.parameterize.sub(/^\d+/, "") + if warnings && idx.blank? + puts "BLANK ID: please put an explicit ID for section #{title}, as in h5(#my-id)" + end + idx + end + end +end diff --git a/guides/rails_guides/kindle.rb b/guides/rails_guides/kindle.rb new file mode 100644 index 0000000000..8a0361ff4c --- /dev/null +++ b/guides/rails_guides/kindle.rb @@ -0,0 +1,116 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +require "kindlerb" +require "nokogiri" +require "fileutils" +require "yaml" +require "date" + +module Kindle + extend self + + def generate(output_dir, mobi_outfile, logfile) + output_dir = File.absolute_path(output_dir) + Dir.chdir output_dir do + puts "=> Using output dir: #{output_dir}" + puts "=> Arranging html pages in document order" + toc = File.read("toc.ncx") + doc = Nokogiri::XML(toc).xpath("//ncx:content", "ncx" => "http://www.daisy.org/z3986/2005/ncx/") + html_pages = doc.select { |c| c[:src] }.map { |c| c[:src] }.uniq + + generate_front_matter(html_pages) + + generate_sections(html_pages) + + generate_document_metadata(mobi_outfile) + + puts "Creating MOBI document with kindlegen. This may take a while." + if Kindlerb.run(output_dir) + puts "MOBI document generated at #{File.expand_path(mobi_outfile, output_dir)}" + end + end + end + + def generate_front_matter(html_pages) + frontmatter = [] + html_pages.delete_if { |x| + if /(toc|welcome|copyright).html/.match?(x) + frontmatter << x unless x =~ /toc/ + true + end + } + html = frontmatter.map { |x| + Nokogiri::HTML(File.open(x)).at("body").inner_html + }.join("\n") + + fdoc = Nokogiri::HTML(html) + fdoc.search("h3").each do |h3| + h3.name = "h4" + end + fdoc.search("h2").each do |h2| + h2.name = "h3" + h2["id"] = h2.inner_text.gsub(/\s/, "-") + end + add_head_section fdoc, "Front Matter" + File.open("frontmatter.html", "w") { |f| f.puts fdoc.to_html } + html_pages.unshift "frontmatter.html" + end + + def generate_sections(html_pages) + FileUtils.rm_rf("sections/") + html_pages.each_with_index do |page, section_idx| + FileUtils.mkdir_p("sections/%03d" % section_idx) + doc = Nokogiri::HTML(File.open(page)) + title = doc.at("title").inner_text.gsub("Ruby on Rails Guides: ", "") + title = page.capitalize.gsub(".html", "") if title.strip == "" + File.open("sections/%03d/_section.txt" % section_idx, "w") { |f| f.puts title } + doc.xpath("//h3[@id]").each_with_index do |h3, item_idx| + subsection = h3.inner_text + content = h3.xpath("./following-sibling::*").take_while { |x| x.name != "h3" }.map(&:to_html) + item = Nokogiri::HTML(h3.to_html + content.join("\n")) + item_path = "sections/%03d/%03d.html" % [section_idx, item_idx] + add_head_section(item, subsection) + item.search("img").each do |img| + img["src"] = "#{Dir.pwd}/#{img['src']}" + end + item.xpath("//li/p").each { |p| p.swap(p.children); p.remove } + File.open(item_path, "w") { |f| f.puts item.to_html } + end + end + end + + def generate_document_metadata(mobi_outfile) + puts "=> Generating _document.yml" + x = Nokogiri::XML(File.open("rails_guides.opf")).remove_namespaces! + cover_jpg = "#{Dir.pwd}/images/rails_guides_kindle_cover.jpg" + cover_gif = cover_jpg.sub(/jpg$/, "gif") + puts `convert #{cover_jpg} #{cover_gif}` + document = { + "doc_uuid" => x.at("package")["unique-identifier"], + "title" => x.at("title").inner_text.gsub(/\(.*$/, " v2"), + "publisher" => x.at("publisher").inner_text, + "author" => x.at("creator").inner_text, + "subject" => x.at("subject").inner_text, + "date" => x.at("date").inner_text, + "cover" => cover_gif, + "masthead" => nil, + "mobi_outfile" => mobi_outfile + } + puts document.to_yaml + File.open("_document.yml", "w") { |f| f.puts document.to_yaml } + end + + def add_head_section(doc, title) + head = Nokogiri::XML::Node.new "head", doc + title_node = Nokogiri::XML::Node.new "title", doc + title_node.content = title + title_node.parent = head + css = Nokogiri::XML::Node.new "link", doc + css["rel"] = "stylesheet" + css["type"] = "text/css" + css["href"] = "#{Dir.pwd}/stylesheets/kindle.css" + css.parent = head + doc.at("body").before head + end +end diff --git a/guides/rails_guides/levenshtein.rb b/guides/rails_guides/levenshtein.rb new file mode 100644 index 0000000000..2213ef754d --- /dev/null +++ b/guides/rails_guides/levenshtein.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +module RailsGuides + module Levenshtein + # This code is based directly on the Text gem implementation. + # Copyright (c) 2006-2013 Paul Battley, Michael Neumann, Tim Fletcher. + # + # Returns a value representing the "cost" of transforming str1 into str2 + def self.distance(str1, str2) + s = str1 + t = str2 + n = s.length + m = t.length + + return m if 0 == n + return n if 0 == m + + d = (0..m).to_a + x = nil + + # avoid duplicating an enumerable object in the loop + str2_codepoint_enumerable = str2.each_codepoint + + str1.each_codepoint.with_index do |char1, i| + e = i + 1 + + str2_codepoint_enumerable.with_index do |char2, j| + cost = (char1 == char2) ? 0 : 1 + x = [ + d[j + 1] + 1, # insertion + e + 1, # deletion + d[j] + cost # substitution + ].min + d[j] = e + e = x + end + + d[m] = x + end + + x + end + end +end diff --git a/guides/rails_guides/markdown.rb b/guides/rails_guides/markdown.rb new file mode 100644 index 0000000000..a98aa8fe66 --- /dev/null +++ b/guides/rails_guides/markdown.rb @@ -0,0 +1,173 @@ +# frozen_string_literal: true + +require "redcarpet" +require "nokogiri" +require "rails_guides/markdown/renderer" + +module RailsGuides + class Markdown + def initialize(view:, layout:, edge:, version:) + @view = view + @layout = layout + @edge = edge + @version = version + @index_counter = Hash.new(0) + @raw_header = "" + @node_ids = {} + end + + def render(body) + @raw_body = body + extract_raw_header_and_body + generate_header + generate_title + generate_body + generate_structure + generate_index + render_page + end + + private + + def dom_id(nodes) + dom_id = dom_id_text(nodes.last.text) + + # Fix duplicate node by prefix with its parent node + if @node_ids[dom_id] + if @node_ids[dom_id].size > 1 + duplicate_nodes = @node_ids.delete(dom_id) + new_node_id = "#{duplicate_nodes[-2][:id]}-#{duplicate_nodes.last[:id]}" + duplicate_nodes.last[:id] = new_node_id + @node_ids[new_node_id] = duplicate_nodes + end + + dom_id = "#{nodes[-2][:id]}-#{dom_id}" + end + + @node_ids[dom_id] = nodes + dom_id + end + + def dom_id_text(text) + escaped_chars = Regexp.escape('\\/`*_{}[]()#+-.!:,;|&<>^~=\'"') + + text.downcase.gsub(/\?/, "-questionmark") + .gsub(/!/, "-bang") + .gsub(/[#{escaped_chars}]+/, " ").strip + .gsub(/\s+/, "-") + end + + def engine + @engine ||= Redcarpet::Markdown.new(Renderer, + no_intra_emphasis: true, + fenced_code_blocks: true, + autolink: true, + strikethrough: true, + superscript: true, + tables: true + ) + end + + def extract_raw_header_and_body + if /^\-{40,}$/.match?(@raw_body) + @raw_header, _, @raw_body = @raw_body.partition(/^\-{40,}$/).map(&:strip) + end + end + + def generate_body + @body = engine.render(@raw_body) + end + + def generate_header + @header = engine.render(@raw_header).html_safe + end + + def generate_structure + @headings_for_index = [] + if @body.present? + @body = Nokogiri::HTML.fragment(@body).tap do |doc| + hierarchy = [] + + doc.children.each do |node| + if /^h[3-6]$/.match?(node.name) + case node.name + when "h3" + hierarchy = [node] + @headings_for_index << [1, node, node.inner_html] + when "h4" + hierarchy = hierarchy[0, 1] + [node] + @headings_for_index << [2, node, node.inner_html] + when "h5" + hierarchy = hierarchy[0, 2] + [node] + when "h6" + hierarchy = hierarchy[0, 3] + [node] + end + + node[:id] = dom_id(hierarchy) unless node[:id] + node.inner_html = "#{node_index(hierarchy)} #{node.inner_html}" + end + end + + doc.css("h3, h4, h5, h6").each do |node| + node.inner_html = "<a class='anchorlink' href='##{node[:id]}'>#{node.inner_html}</a>" + end + end.to_html + end + end + + def generate_index + if @headings_for_index.present? + raw_index = "" + @headings_for_index.each do |level, node, label| + if level == 1 + raw_index += "1. [#{label}](##{node[:id]})\n" + elsif level == 2 + raw_index += " * [#{label}](##{node[:id]})\n" + end + end + + @index = Nokogiri::HTML.fragment(engine.render(raw_index)).tap do |doc| + doc.at("ol")[:class] = "chapters" + end.to_html + + @index = <<-INDEX.html_safe + <div id="subCol"> + <h3 class="chapter"><img src="images/chapters_icon.gif" alt="" />Chapters</h3> + #{@index} + </div> + INDEX + end + end + + def generate_title + if heading = Nokogiri::HTML.fragment(@header).at(:h2) + @title = "#{heading.text} — Ruby on Rails Guides" + else + @title = "Ruby on Rails Guides" + end + end + + def node_index(hierarchy) + case hierarchy.size + when 1 + @index_counter[2] = @index_counter[3] = @index_counter[4] = 0 + "#{@index_counter[1] += 1}" + when 2 + @index_counter[3] = @index_counter[4] = 0 + "#{@index_counter[1]}.#{@index_counter[2] += 1}" + when 3 + @index_counter[4] = 0 + "#{@index_counter[1]}.#{@index_counter[2]}.#{@index_counter[3] += 1}" + when 4 + "#{@index_counter[1]}.#{@index_counter[2]}.#{@index_counter[3]}.#{@index_counter[4] += 1}" + end + end + + def render_page + @view.content_for(:header_section) { @header } + @view.content_for(:page_title) { @title } + @view.content_for(:index_section) { @index } + @view.render(layout: @layout, html: @body.html_safe) + end + end +end diff --git a/guides/rails_guides/markdown/renderer.rb b/guides/rails_guides/markdown/renderer.rb new file mode 100644 index 0000000000..f186ac526f --- /dev/null +++ b/guides/rails_guides/markdown/renderer.rb @@ -0,0 +1,128 @@ +# frozen_string_literal: true + +module RailsGuides + class Markdown + class Renderer < Redcarpet::Render::HTML + cattr_accessor :edge, :version + + def block_code(code, language) + <<-HTML +<div class="code_container"> +<pre class="brush: #{brush_for(language)}; gutter: false; toolbar: false"> +#{ERB::Util.h(code)} +</pre> +</div> +HTML + end + + def link(url, title, content) + if url.start_with?("http://api.rubyonrails.org") + %(<a href="#{api_link(url)}">#{content}</a>) + elsif title + %(<a href="#{url}" title="#{title}">#{content}</a>) + else + %(<a href="#{url}">#{content}</a>) + end + end + + def header(text, header_level) + # Always increase the heading level by 1, so we can use h1, h2 heading in the document + header_level += 1 + + header_with_id = text.scan(/(.*){#(.*)}/) + unless header_with_id.empty? + %(<h#{header_level} id="#{header_with_id[0][1].strip}">#{header_with_id[0][0].strip}</h#{header_level}>) + else + %(<h#{header_level}>#{text}</h#{header_level}>) + end + end + + def paragraph(text) + if text =~ %r{^NOTE:\s+Defined\s+in\s+<code>(.*?)</code>\.?$} + %(<div class="note"><p>Defined in <code><a href="#{github_file_url($1)}">#{$1}</a></code>.</p></div>) + elsif /^(TIP|IMPORTANT|CAUTION|WARNING|NOTE|INFO|TODO)[.:]/.match?(text) + convert_notes(text) + elsif text.include?("DO NOT READ THIS FILE ON GITHUB") + elsif text =~ /^\[<sup>(\d+)\]:<\/sup> (.+)$/ + linkback = %(<a href="#footnote-#{$1}-ref"><sup>#{$1}</sup></a>) + %(<p class="footnote" id="footnote-#{$1}">#{linkback} #{$2}</p>) + else + text = convert_footnotes(text) + "<p>#{text}</p>" + end + end + + private + + def convert_footnotes(text) + text.gsub(/\[<sup>(\d+)\]<\/sup>/i) do + %(<sup class="footnote" id="footnote-#{$1}-ref">) + + %(<a href="#footnote-#{$1}">#{$1}</a></sup>) + end + end + + def brush_for(code_type) + case code_type + when "ruby", "sql", "plain" + code_type + when "erb", "html+erb" + "ruby; html-script: true" + when "html" + "xml" # HTML is understood, but there are .xml rules in the CSS + else + "plain" + end + end + + def convert_notes(body) + # The following regexp detects special labels followed by a + # paragraph, perhaps at the end of the document. + # + # It is important that we do not eat more than one newline + # because formatting may be wrong otherwise. For example, + # if a bulleted list follows, the first item is not rendered + # as a list item, but as a paragraph starting with a plain + # asterisk. + body.gsub(/^(TIP|IMPORTANT|CAUTION|WARNING|NOTE|INFO|TODO)[.:](.*?)(\n(?=\n)|\Z)/m) do + css_class = \ + case $1 + when "CAUTION", "IMPORTANT" + "warning" + when "TIP" + "info" + else + $1.downcase + end + %(<div class="#{css_class}"><p>#{$2.strip}</p></div>) + end + end + + def github_file_url(file_path) + tree = version || edge + + root = file_path[%r{(\w+)/}, 1] + path = \ + case root + when "abstract_controller", "action_controller", "action_dispatch" + "actionpack/lib/#{file_path}" + when /\A(action|active)_/ + "#{root.sub("_", "")}/lib/#{file_path}" + else + file_path + end + + "https://github.com/rails/rails/tree/#{tree}/#{path}" + end + + def api_link(url) + if %r{http://api\.rubyonrails\.org/v\d+\.}.match?(url) + url + elsif edge + url.sub("api", "edgeapi") + else + url.sub(/(?<=\.org)/, "/#{version}") + end + end + end + end +end |