|
#!/bin/awk -f |
|
# |
|
# by: Jesus Galan (yiyus) 2009 |
|
# |
|
# Usage: md2html.awk file.md > file.html |
|
# See: http://4l77.com/src/md2html.awk |
|
|
|
# Attribute values need quotes escaped as well as the usual entities. |
|
# sanitize_url <url> |
|
# Empty for any scheme that executes, so a link cannot run code. |
|
# |
|
# href="javascript:..." is a script the reader triggers by clicking, and |
|
# data: and vbscript: and file: are the same problem wearing other names. |
|
# Nothing in a document written as prose needs them, and a parser whose |
|
# output is served to other people cannot tell a deliberate one from an |
|
# injected one. |
|
# |
|
# The check is made on a throwaway copy, never on the URL that is emitted. |
|
# The copy is lowercased, stripped of leading control characters and |
|
# whitespace, has the HTML entities that could hide a colon decoded, and is |
|
# percent-decoded until no escape remains. A browser decodes an href before it |
|
# decides what scheme it holds, so the check has to read what the browser will |
|
# read: "%6a%61%76%61%73%63%72%69%70%74:", "%6aavascript:" and the |
|
# double-encoded "%256a..." are all javascript: written to slip past a test |
|
# that reads the raw text. |
|
# |
|
# Only the copy is decoded. A legitimate %20 in a path must survive into the |
|
# output untouched, and it does, because the URL that is returned is the one |
|
# that came in -- once a scheme is rejected the whole URL is dropped anyway. |
|
function pct_decode(t, out, lo, hi, HEX) { |
|
# One pass of %xx -> the byte it names, without strtonum: that is a gawk |
|
# extension and this has to run under any awk. index() into a table of |
|
# the sixteen digits gives the nibble values, and index() is |
|
# case-sensitive, so the digits are folded before the lookup rather than |
|
# the table being widened. |
|
# |
|
# The pair is matched in either case even though the caller folds the |
|
# whole string first: a decoder that only reads %6a and not %6A is wrong |
|
# on its own terms, whatever its callers happen to do. |
|
HEX = "0123456789abcdef" |
|
out = "" |
|
while (match(t, /%[0-9a-fA-F][0-9a-fA-F]/)) { |
|
hi = index(HEX, tolower(substr(t, RSTART + 1, 1))) - 1 |
|
lo = index(HEX, tolower(substr(t, RSTART + 2, 1))) - 1 |
|
out = out substr(t, 1, RSTART - 1) sprintf("%c", hi * 16 + lo) |
|
t = substr(t, RSTART + 3) |
|
} |
|
return out t |
|
} |
|
|
|
function sanitize_url(u, probe, prev) { |
|
probe = tolower(u) |
|
gsub(/[\000-\040]/, "", probe) |
|
gsub(/�*9;|�*10;|�*13;|&tab;|&newline;/, "", probe) |
|
gsub(/�*58;|:/, ":", probe) |
|
gsub(/�*x?[0-9a-f]+;?/, "", probe) |
|
# Until it stops changing, so "%256a" -- a percent sign that is itself |
|
# encoded -- is decoded the second time round rather than left half done. |
|
# |
|
# Folded again after each pass, not only at the start. Decoding reveals |
|
# bytes that were not in the text to fold the first time: "%4a" is |
|
# lowercase already and decodes to "J", so "%4Aavascript:" became |
|
# "Javascript:" -- past a check written in lowercase, and a scheme a |
|
# browser reads case-insensitively. The fold has to follow the decode |
|
# that produced the letter. |
|
do { |
|
prev = probe |
|
probe = tolower(pct_decode(probe)) |
|
gsub(/[\000-\040]/, "", probe) |
|
} while (probe != prev) |
|
if (probe ~ /^(javascript|data|vbscript|file):/) |
|
return "" |
|
return u |
|
} |
|
|
|
# close_quotes |
|
# Emits the closing tag for any blockquote still open, and forgets it. |
|
# |
|
# The rules that build a fence or a table finish with next, which returns |
|
# before the loop further down that closes open blocks -- so a fence or a |
|
# table written after a blockquote was swallowed into it: the paragraph |
|
# closed, the quote did not, and the block rendered inside the quote it should |
|
# have followed. Those rules close it themselves, here, rather than leaving a |
|
# level set for a loop they never reach. |
|
function close_quotes( i) { |
|
for(i = nl; i > 0 && block[i] == "blockquote"; i--) |
|
oprint("</blockquote>"); |
|
if(i < nl) |
|
nl = i; |
|
} |
|
|
|
function escattr(t) { |
|
t = eschtml(t); |
|
gsub(">", "\\>", t); |
|
gsub("\"", "\\"", t); |
|
return t; |
|
} |
|
|
|
# Fenced content is fully literal, so > is escaped too. eschtml is shared with |
|
# inline parsing, where a bare > is legitimate text, so it is left alone. |
|
function escfence(t) { |
|
t = eschtml(t); |
|
gsub(">", "\\>", t); |
|
return t; |
|
} |
|
|
|
function eschtml(t) { |
|
gsub("&", "\\&", t); |
|
gsub("<", "\\<", t); |
|
return t; |
|
} |
|
|
|
function oprint(t){ |
|
if(nr == 0) |
|
print t; |
|
else |
|
otext = otext "\n" t; |
|
} |
|
|
|
function subref(id, rep){ |
|
# The stored value is already escaped, so it can contain " -- and |
|
# sub() reads & in a replacement as "the text that matched", which would |
|
# splice the placeholder back into its own replacement. Doubling the |
|
# ampersands makes sub() emit them literally. |
|
rep = ref[id] |
|
gsub(/&/, "\\\\&", rep) |
|
for(; nr > 0 && sub("<<" id, rep, otext); nr--); |
|
if(nr == 0 && otext) { |
|
print otext; |
|
otext = ""; |
|
} |
|
} |
|
|
|
# Escapes regex metacharacters so a literal string can be interpolated into |
|
# a dynamic regex. Without this "^" "*" reaches the regex engine, which mawk |
|
# rejects outright and gawk quietly mismatches. |
|
function re_quote(s, out, i, c) { |
|
out = ""; |
|
for(i = 1; i <= length(s); i++) { |
|
c = substr(s, i, 1); |
|
if(index("\\^$.[]|()*+?{}/", c) > 0) |
|
out = out "\\" c; |
|
else |
|
out = out c; |
|
} |
|
return out; |
|
} |
|
|
|
function nextil(t) { |
|
if(!match(t, /[`<&\[*_~\\-]|(!\[)|(\[\^)|(\[#)/)) |
|
return t; |
|
t1 = substr(t, 1, RSTART - 1); |
|
tag = substr(t, RSTART, RLENGTH); |
|
t2 = substr(t, RSTART + RLENGTH); |
|
if(ilcode && tag != "`") |
|
return eschtml(t1 tag) nextil(t2); |
|
# Backslash escaping |
|
if(tag == "\\"){ |
|
if(match(t2, /^[\\`*_{}\[\]()#+\-\.!]/)){ |
|
tag = substr(t2, 1, 1); |
|
t2 = substr(t2, 2); |
|
} |
|
return t1 tag nextil(t2); |
|
} |
|
# Dashes |
|
if(tag == "-"){ |
|
if(sub(/^-/, "", t2)) |
|
tag = "—"; |
|
return t1 tag nextil(t2); |
|
} |
|
# Inline Code |
|
if(tag == "`"){ |
|
if(sub(/^`/, "", t2)){ |
|
if(!match(t2, /``/)) |
|
return t1 "”" nextil(t2); |
|
ilcode2 = !ilcode2; |
|
} |
|
else if(ilcode2) |
|
return t1 tag nextil(t2); |
|
tag = "<code>"; |
|
if(ilcode){ |
|
t1 = eschtml(t1); |
|
tag = "</code>"; |
|
} |
|
ilcode = !ilcode; |
|
return t1 tag nextil(t2); |
|
} |
|
if(tag == "<"){ |
|
# Autolinks |
|
if(match(t2, /^[^ ]+[\.@][^ ]+>/)){ |
|
url = eschtml(substr(t2, 1, RLENGTH - 1)); |
|
t2 = substr(t2, RLENGTH + 1); |
|
linktext = url; |
|
if(match(url, /@/) && !match(url, /^mailto:/)) |
|
url = "mailto:" url; |
|
return t1 "<a href=\"" escattr(sanitize_url(url)) "\">" linktext "</a>" nextil(t2); |
|
} |
|
# Html tags |
|
# Passed through only when the caller has asked for it. A parser whose |
|
# output is served to other people cannot tell an author's deliberate |
|
# <div> from an injected <script>, and the safe reading has to be the |
|
# one you get without saying anything. |
|
if(match(t2, /^[A-Za-z\/!][^>]*>/)){ |
|
tag = tag substr(t2, RSTART, RLENGTH); |
|
t2 = substr(t2, RSTART + RLENGTH); |
|
if(UNSAFE_HTML) |
|
return t1 tag nextil(t2); |
|
# escattr, not eschtml: eschtml leaves > alone because a bare > |
|
# is ordinary in prose, but here the whole tag is being shown as |
|
# text and a half-escaped tag reads as broken markup. |
|
return t1 escattr(tag) nextil(t2); |
|
} |
|
return t1 "<" nextil(t2); |
|
} |
|
# Html special entities |
|
if(tag == "&"){ |
|
if(match(t2, /^#?[A-Za-z0-9]+;/)){ |
|
tag = tag substr(t2, RSTART, RLENGTH); |
|
t2 = substr(t2, RLENGTH + 1); |
|
return t1 tag nextil(t2); |
|
} |
|
return t1 "&" nextil(t2); |
|
} |
|
# Images |
|
if(tag == "!["){ |
|
if(!match(t2, /(\[.*\])|(\(.*\))/)) |
|
return t1 tag nextil(t2); |
|
match(t2, /^[^\]]*/); |
|
alt = substr(t2, 1, RLENGTH); |
|
t2 = substr(t2, RLENGTH + 2); |
|
if(match(t2, /^\(/)){ |
|
# Inline |
|
sub(/^\(/, "", t2); |
|
match(t2, /^[^\)]+/); |
|
url = eschtml(substr(t2, 1, RLENGTH)); |
|
t2 = substr(t2, RLENGTH + 2); |
|
title = ""; |
|
if(match(url, /[ ]+".*"[ ]*$/)) { |
|
title = substr(url, RSTART, RLENGTH); |
|
url = substr(url, 1, RSTART - 1); |
|
match(title, /".*"/); |
|
title = " title=\"" escattr(substr(title, RSTART + 1, RLENGTH - 2)) "\""; |
|
} |
|
if(match(url, /^<.*>$/)) |
|
url = substr(url, 2, RLENGTH - 2); |
|
return t1 "<img src=\"" escattr(sanitize_url(url)) "\" alt=\"" escattr(alt) "\"" title " />" nextil(t2); |
|
} |
|
else{ |
|
# Referenced |
|
sub(/^ ?\[/, "", t2); |
|
id = alt; |
|
if(match(t2, /^[^\]]+/)) |
|
id = substr(t2, 1, RLENGTH); |
|
t2 = substr(t2, RLENGTH + 2); |
|
if(ref[id]) |
|
r = ref[id]; |
|
else{ |
|
r = "<<" id; |
|
nr++; |
|
} |
|
# r is not escaped here: it may be the "<<id" placeholder that |
|
# subref() fills in when the definition is read further down the |
|
# document, and escaping mangles the marker so the substitution |
|
# never matches -- a referenced image rendered with a literal |
|
# "<<id" for its src. Escaping the definition itself is |
|
# part of the attribute-safety work, not this line. |
|
return t1 "<img src=\"" r "\" alt=\"" escattr(alt) "\" />" nextil(t2); |
|
} |
|
} |
|
# Citations |
|
# |
|
# [#id] or [locator][#id]. Numbered by first use; a repeat reuses the |
|
# number, matching multimarkdown. |
|
if(tag == "[#"){ |
|
match(t2, /^[^\]]*/); |
|
cid = substr(t2, 1, RLENGTH); |
|
t2 = substr(t2, RLENGTH + 2); |
|
if(!(cid in cite_num)){ |
|
cite_num[cid] = ++cite_n; |
|
cite_order[cite_n] = cid; |
|
} |
|
cnum = cite_num[cid]; |
|
# A locator set by the link branch for the [locator][#id] form. |
|
cloc = cite_locator; |
|
cite_locator = ""; |
|
clabel = (cloc != "") ? "(" cloc ", " cnum ")" : "(" cnum ")"; |
|
return t1 "<a href=\"#cn:" cnum "\" id=\"cnref:" cnum "\" data-citation=\"" escattr(cid) \ |
|
"\" title=\"see citation\" class=\"citation\">" clabel "</a>" nextil(t2); |
|
} |
|
|
|
# Footnotes |
|
if(tag == "[^"){ |
|
match(t2, /^[^\]]*(\[[^\]]*\][^\]]*)*/); |
|
linktext = substr(t2, 1, RLENGTH); |
|
t2 = substr(t2, RLENGTH + 2); |
|
return t1 "<sup class=\"fnref\" data-fnref=\"" escattr(linktext) "\"><a href=\"#fn-" escattr(linktext) "\" id=\"fnref-" escattr(linktext) "\">" eschtml(linktext) "</a></sup>" nextil(t2); |
|
} |
|
# Links |
|
if(tag == "["){ |
|
# A shortcut reference is a bare [text] with no following [] or (). |
|
# It is only a link if a definition matching the text exists, so the |
|
# early return here must not fire for one -- it is decided below. |
|
if(!match(t2, /(\[.*\])|(\(.*\))/)){ |
|
if(!match(t2, /^[^\]]*\]/)) |
|
return t1 tag nextil(t2); |
|
shortcut_candidate = 1; |
|
} |
|
else |
|
shortcut_candidate = 0; |
|
match(t2, /^[^\]]*(\[[^\]]*\][^\]]*)*/); |
|
linktext = substr(t2, 1, RLENGTH); |
|
t2 = substr(t2, RLENGTH + 2); |
|
if(match(t2, /^\(/)){ |
|
# Inline |
|
match(t2, /^[^\)]+(\([^\)]+\)[^\)]*)*/); |
|
url = substr(t2, 2, RLENGTH - 1); |
|
pt2 = substr(t2, RLENGTH + 2); |
|
title = ""; |
|
if(match(url, /[ ]+".*"[ ]*$/)) { |
|
title = substr(url, RSTART, RLENGTH); |
|
url = substr(url, 1, RSTART - 1); |
|
match(title, /".*"/); |
|
title = " title=\"" escattr(substr(title, RSTART + 1, RLENGTH - 2)) "\""; |
|
} |
|
if(match(url, /^<.*>$/)) |
|
url = substr(url, 2, RLENGTH - 2); |
|
url = eschtml(url); |
|
return t1 "<a href=\"" escattr(sanitize_url(url)) "\"" title ">" nextil(linktext) "</a>" nextil(pt2); |
|
} |
|
else{ |
|
# Referenced: [text][id], [text][] or the shortcut [text]. |
|
id = linktext; |
|
shortcut = 1; |
|
if(match(t2, /^ ?\[#/)){ |
|
# [locator][#id] is a citation, not a reference link. The |
|
# locator is handed to the citation branch directly; nextil |
|
# has already split t1 away, so it cannot be recovered there. |
|
sub(/^ ?\[/, "", t2); |
|
cite_locator = linktext; |
|
return t1 nextil("[" t2); |
|
} |
|
if(match(t2, /^ ?\[/)){ |
|
shortcut = 0; |
|
sub(/^ ?\[/, "", t2); |
|
if(match(t2, /^[^\]]*/)){ |
|
# An empty [] means "reuse the link text as the label". |
|
if(RLENGTH > 0) |
|
id = substr(t2, 1, RLENGTH); |
|
t2 = substr(t2, RLENGTH + 2); |
|
} |
|
} |
|
id = tolower(id); |
|
if(ref[id]) |
|
r = ref[id]; |
|
else{ |
|
# The definition may appear later in the document, so the |
|
# link is emitted with a placeholder that subref() fills in |
|
# when the definition is read. A shortcut [text] that is |
|
# still unresolved at EOF is restored to literal text by |
|
# the END rule. |
|
r = "<<" id; |
|
nr++; |
|
if(shortcut) |
|
shortcut_ref[id] = linktext; |
|
} |
|
pt2 = t2; |
|
return t1 "<a href=\"" r "\">" nextil(linktext) "</a>" nextil(pt2); |
|
} |
|
} |
|
# Strikethrough |
|
# |
|
# Only the doubled form, which is what GFM defines and what authors |
|
# write. A single ~ stays literal so that "~/path" and "approx ~5" are |
|
# not silently turned into markup. An unpaired "~~" is left alone rather |
|
# than opening a <del> that would run to the end of the document. |
|
if(tag == "~"){ |
|
if(sub(/^~/, "", t2)){ |
|
if(match(t2, /~~/)){ |
|
strike = substr(t2, 1, RSTART - 1); |
|
# t1, tag and t2 are globals shared with every nextil frame, |
|
# so parsing the struck text clobbers the tail still waiting |
|
# here -- it came back with whatever the inner call left |
|
# behind, dropping the rest of the line. The link handler |
|
# above keeps its own copy for the same reason. |
|
strike_t1 = t1; |
|
strike_rest = substr(t2, RSTART + 2); |
|
strike_done = nextil(strike); |
|
return strike_t1 "<del>" strike_done "</del>" nextil(strike_rest); |
|
} |
|
return t1 "~~" nextil(t2); |
|
} |
|
return t1 "~" nextil(t2); |
|
} |
|
# Emphasis |
|
if(match(tag, /[*_]/)){ |
|
# An underscore between two word characters is literal, not an |
|
# emphasis marker -- otherwise identifiers like node_modules and |
|
# snake_case_name get torn apart, and an unmatched one leaves a |
|
# stray <em> open for the rest of the document. CommonMark makes |
|
# the same exception, and only for _: intraword * still emphasises. |
|
if(tag == "_" && match(t1, /[A-Za-z0-9]$/) && match(t2, /^[A-Za-z0-9]/)) |
|
return t1 tag nextil(t2); |
|
# An underscore that is the whole content of an HTML element is |
|
# literal: <span class="us">_</span> is a way of writing a visible |
|
# underscore, not emphasis. Without this it opened an <em> that |
|
# swallowed the markup after it, and two such underscores paired with |
|
# each other across the tags between them, emitting mis-nested |
|
# "<b><em></b> ... <b></em></b>". CommonMark reaches the same result |
|
# by its flanking rules: a delimiter preceded by punctuation and |
|
# followed by punctuation is neither opening nor closing. |
|
if(tag == "_" && t1 == "" && match(t2, /^</) && stag[ns] != "_" && stag[ns] != "__") |
|
return t1 tag nextil(t2); |
|
ntag = tag; |
|
# `tag` is a literal * or _, so it must be escaped before being |
|
# interpolated into a regex: "^" "*" is not a valid regex. |
|
etag = "^" re_quote(tag); |
|
if(sub(etag, "", t2)){ |
|
if(stag[ns] == tag && match(t2, etag)) |
|
t2 = tag t2; |
|
else |
|
ntag = tag tag |
|
} |
|
n = length(ntag); |
|
tag = (n == 2) ? "strong" : "em"; |
|
if(match(t1, / $/) && match(t2, /^ /)) |
|
return t1 tag nextil(t2); |
|
if(stag[ns] == ntag){ |
|
tag = "/" tag; |
|
ns--; |
|
} |
|
else |
|
stag[++ns] = ntag; |
|
tag = "<" tag ">"; |
|
return t1 tag nextil(t2); |
|
} |
|
} |
|
|
|
function inline(t) { |
|
ilcode = 0; |
|
ilcode2 = 0; |
|
ns = 0; |
|
|
|
return nextil(t); |
|
} |
|
|
|
# heading_id <text> |
|
# The slug a heading is addressable by: lowercased, runs of anything that is |
|
# not a letter or digit collapsed to a single dash, ends trimmed. |
|
# |
|
# A document long enough to link within needs its headings to be targets. Any |
|
# "#section" link -- from its own table of contents, from a sibling page, from |
|
# somewhere else entirely -- resolves to nothing without them. |
|
# |
|
# Inline markup is stripped first: a heading may be emphasised or hold code, |
|
# and "<code>Vision</code>" must still slug to "vision". |
|
# |
|
# Collisions are left alone. Two headings with the same words produce the same |
|
# id and the first wins, which is what a reader following a link expects and |
|
# what every other implementation does; numbering them would make the ids |
|
# depend on how many times a word appears earlier in the document. |
|
function heading_id(t, s) { |
|
s = t |
|
gsub(/<[^>]*>/, "", s) |
|
s = tolower(s) |
|
gsub(/&[a-z]+;|&#[0-9]+;|&#x[0-9a-fA-F]+;/, "", s) |
|
gsub(/[^a-z0-9]+/, "-", s) |
|
sub(/^-+/, "", s) |
|
sub(/-+$/, "", s) |
|
return s |
|
} |
|
|
|
# para_id <html> |
|
# The anchor a paragraph is addressable by: "p-" and a hash of the paragraph's |
|
# own text. |
|
# |
|
# Derived from the content rather than the position, so a permalink survives |
|
# edits elsewhere on the page -- insert three paragraphs above this one, or |
|
# reorder the sections around it, and the link still resolves. It breaks only |
|
# when this paragraph's own wording changes, which is the honest failure: the |
|
# text someone linked to is not there any more. A positional p-7 would keep |
|
# resolving after an insertion and quietly point at the wrong paragraph, which |
|
# is worse than breaking. |
|
# |
|
# Inline markup is stripped first so that editing a link's target, or wrapping |
|
# a word in emphasis, does not move the anchor. |
|
function para_id(t, s) { |
|
s = t |
|
gsub(/<[^>]*>/, "", s) |
|
gsub(/&[a-z]+;|&#[0-9]+;|&#x[0-9a-fA-F]+;/, "", s) |
|
gsub(/[ \t]+/, " ", s) |
|
sub(/^ /, "", s); sub(/ $/, "", s) |
|
if (s == "") return "" |
|
return "p-" fnv1a(s) |
|
} |
|
|
|
# fnv1a <string> |
|
# FNV-1a, 32 bits, in awk rather than by calling out to a hash program: a page |
|
# has hundreds of paragraphs and a sub-process each would be the slowest part |
|
# of the build. Checked against every paragraph in the site at the time it was |
|
# written -- no collisions between different texts. |
|
function fnv1a(s, h, i, c, n) { |
|
h = 2166136261 |
|
n = length(s) |
|
for (i = 1; i <= n; i++) { |
|
c = index(HASHCHARS, substr(s, i, 1)) |
|
if (c == 0) c = 32 |
|
h = xor32(h, c) |
|
h = (h * 16777619) % 4294967296 |
|
} |
|
return sprintf("%08x", h) |
|
} |
|
|
|
function xor32(a, b, r, i, x, y) { |
|
r = 0; i = 1 |
|
while (a > 0 || b > 0) { |
|
x = a % 2; y = b % 2 |
|
if (x != y) r += i |
|
a = int(a / 2); b = int(b / 2); i *= 2 |
|
} |
|
return r |
|
} |
|
|
|
function printp(tag, id) { |
|
if(!match(text, /^[ ]*$/)){ |
|
text = inline(text); |
|
if(tag != ""){ |
|
# Headings carry an id; nothing else does. |
|
if(tag ~ /^h[1-6]$/){ |
|
id = heading_id(text) |
|
if(id != "") |
|
oprint("<" tag " id=\"" id "\">" text "</" tag ">"); |
|
else |
|
oprint("<" tag ">" text "</" tag ">"); |
|
} |
|
else if(tag == "p"){ |
|
# Two paragraphs with the same words hash the same; the second |
|
# and later take a suffix so every id on the page is unique. |
|
id = para_id(text) |
|
if(id != ""){ |
|
PARA_SEEN[id]++ |
|
if(PARA_SEEN[id] > 1) |
|
id = id "-" PARA_SEEN[id] |
|
oprint("<p id=\"" id "\">" text "</p>"); |
|
} |
|
else |
|
oprint("<p>" text "</p>"); |
|
} |
|
else |
|
oprint("<" tag ">" text "</" tag ">"); |
|
} |
|
else |
|
oprint(text); |
|
} |
|
text = ""; |
|
} |
|
|
|
BEGIN { |
|
# Character table for fnv1a: index() gives each byte a value, so the hash |
|
# needs no ord() -- which one-true-awk does not have. |
|
HASHCHARS = " !\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~" |
|
blank = 0; |
|
code = 0; |
|
hr = 0; |
|
html = 0; |
|
nl = 0; |
|
nr = 0; |
|
otext = ""; |
|
text = ""; |
|
par = "p"; |
|
} |
|
|
|
# References |
|
# Reference link definitions: [id]: url "optional title" |
|
# Standard markdown. The [^!id] form below predates this and is kept so any |
|
# document using it still resolves. Labels are case-insensitive, so they are |
|
# normalised to lowercase on both definition and use. |
|
!code && !fence && /^ *\[[^\^#\]][^\]]*\]:[ ]+/ { |
|
sub(/^ *\[/, ""); |
|
match($0, /\]/); |
|
id = tolower(substr($0, 1, RSTART - 1)); |
|
sub(/^[^\]]*\]:[ ]+/, ""); |
|
title = ""; |
|
if(match($0, /".*"$/)) |
|
title = "\" title=\"" escattr(substr($0, RSTART + 1, RLENGTH - 2)); |
|
sub(/[ ]+".*"$/, ""); |
|
# The url lands in an href or src, so it is escaped as an attribute and |
|
# filtered for schemes that execute -- a reference definition is as good |
|
# a place to hide a javascript: url as an inline link is. |
|
url = escattr(sanitize_url($0)); |
|
ref[id] = url title; |
|
|
|
subref(id); |
|
next; |
|
} |
|
|
|
!code && /^ *\[\^![^\]]*\]:[ ]+/ { |
|
sub(/^ *\[\^!/, ""); |
|
match($0, /\]/); |
|
id = substr($0, 1, RSTART - 1); |
|
sub(id "\\]:[ ]+", ""); |
|
id = tolower(id); |
|
title = ""; |
|
if(match($0, /".*"$/)) |
|
title = "\" title=\"" escattr(substr($0, RSTART + 1, RLENGTH - 2)); |
|
sub(/[ ]+".*"$/, ""); |
|
# The url lands in an href or src, so it is escaped as an attribute and |
|
# filtered for schemes that execute -- a reference definition is as good |
|
# a place to hide a javascript: url as an inline link is. |
|
url = escattr(sanitize_url($0)); |
|
ref[id] = url title; |
|
|
|
subref(id); |
|
next; |
|
} |
|
|
|
!code && !fence && /^ *\[#[^\]]*\]:[ ]+/ { |
|
sub(/^ *\[#/, ""); |
|
match($0, /\]/); |
|
id = substr($0, 1, RSTART - 1); |
|
sub(/^[^\]]*\]:[ ]+/, ""); |
|
cite_body[id] = $0; |
|
next; |
|
} |
|
|
|
!code && /^ *\[\^[^\]]*\]:[ ]+/ { |
|
sub(/^ *\[\^/, ""); |
|
match($0, /\]/); |
|
id = substr($0, 1, RSTART - 1); |
|
sub(id "\\]:[ ]+", ""); |
|
sub(/[ ]+".*"$/, ""); |
|
# Stored raw, like a citation's body: inline() escapes what needs it and |
|
# resolves what does not. Pre-escaping turned every "<" into "<" before |
|
# the parser could see it, so an autolink -- <https://example.com> -- came |
|
# out as literal text in a footnote while working in prose and in |
|
# citations. The variable was called "url" because this rule began life |
|
# handling link-reference definitions, where a bare URL was all it held. |
|
fnref[id] = $0; |
|
|
|
subref(id); |
|
next; |
|
} |
|
|
|
# html |
|
# Block-level raw HTML, passed through only when the caller has asked for it |
|
# with -v UNSAFE_HTML=1. Without the flag the line is not recognised as HTML |
|
# at all and falls through to ordinary paragraph handling, which escapes it -- |
|
# so a document that opens with <script> renders that tag as text rather than |
|
# running it. |
|
UNSAFE_HTML && !html && /^<(address|blockquote|center|dir|div|dl|fieldset|form|h[1-6r]|\ |
|
isindex|menu|noframes|noscript|ol|p|pre|table|ul|!--)/ { |
|
if(code) |
|
oprint(code_held "</code></pre>"); code_held = ""; |
|
for(; !text && block[nl] == "blockquote"; nl--) |
|
oprint("</blockquote>"); |
|
match($0, /^<(address|blockquote|center|dir|div|dl|fieldset|form|h[1-6r]|\ |
|
isindex|menu|noframes|noscript|ol|p|pre|table|ul|!--)/); |
|
htag = substr($0, 2, RLENGTH - 1); |
|
if(!match($0, "(<\\/" htag ">)|((^<hr ?\\/?)|(--)>$)")) |
|
html = 1; |
|
if(html && match($0, /^<hr/)) |
|
hr = 1; |
|
oprint($0); |
|
next; |
|
} |
|
|
|
UNSAFE_HTML && html && (/(^<\/(address|blockquote|center|dir|div|dl|fieldset|form|h[1-6r]|\ |
|
isindex|menu|noframes|noscript|ol|p|pre|table|ul).*)|(--)>$/ || |
|
(hr && />$/)) { |
|
html = 0; |
|
hr = 0; |
|
oprint($0); |
|
next; |
|
} |
|
|
|
html { |
|
oprint($0); |
|
next; |
|
} |
|
|
|
# MultiMarkdown metadata block |
|
# |
|
# Key: value pairs at the very top of the file, ending at the first blank |
|
# line. The pairs are preserved verbatim rather than stripped -- the author |
|
# wrote them deliberately -- but each is wrapped so a theme can style or hide |
|
# it, and so the caller can read the values back. Whether they are visible is |
|
# a readability decision for the consumer, not the parser's to make. |
|
# |
|
# The entire leading run must look like metadata. If any line before the first |
|
# blank is not Key: value, the whole block is ordinary prose -- otherwise a |
|
# post opening "Note: something" would silently become metadata. |
|
# |
|
# Lines are buffered rather than consumed with getline, so the main rules see |
|
# them unchanged when the block turns out to be prose. |
|
meta_state == 0 { |
|
if(NR == 1 && $0 ~ /^[A-Za-z][A-Za-z0-9 _-]*:[ \t]*./){ |
|
meta_state = 1; |
|
meta_n = 0; |
|
meta_buf[++meta_n] = $0; |
|
next; |
|
} |
|
meta_state = 2; |
|
} |
|
|
|
meta_state == 1 { |
|
if($0 ~ /^[ \t]*$/){ |
|
# End of the block, and every line matched: emit them as metadata. |
|
for(mi = 1; mi <= meta_n; mi++){ |
|
mline = meta_buf[mi]; |
|
match(mline, /:/); |
|
mkey = substr(mline, 1, RSTART - 1); |
|
mval = substr(mline, RSTART + 1); |
|
sub(/^[ \t]+/, "", mval); |
|
sub(/[ \t]+$/, "", mval); |
|
oprint("<p class=\"meta\" data-key=\"" escattr(mkey) \ |
|
"\" data-value=\"" escattr(mval) "\">" eschtml(mline) "</p>"); |
|
} |
|
meta_state = 2; |
|
next; |
|
} |
|
if($0 ~ /^[A-Za-z][A-Za-z0-9 _-]*:[ \t]*./){ |
|
meta_buf[++meta_n] = $0; |
|
next; |
|
} |
|
# Not a metadata block after all. The buffered lines are ordinary prose, |
|
# and awk cannot re-run its own rules over them, so they are pushed back |
|
# into the paragraph accumulator that the fall-through rule uses. That is |
|
# where a run of plain lines would have ended up anyway. |
|
meta_state = 2; |
|
for(mi = 1; mi <= meta_n; mi++) |
|
text = (text ? text " " : "") eschtml(meta_buf[mi]); |
|
meta_n = 0; |
|
} |
|
|
|
# Tables (MultiMarkdown) |
|
# |
|
# A row is only a table row if the line after it is a separator: |---|:-:|. |
|
# That means one line of lookahead, which is done by holding the candidate |
|
# header until the next line decides. Alignment comes from the colons in the |
|
# separator and is emitted as an inline style, matching multimarkdown's own |
|
# output. |
|
|
|
function table_cells(line, n, i, out) { |
|
# Split on unescaped pipes, dropping the optional edge pipes. |
|
sub(/^[ \t]*\|/, "", line); |
|
sub(/\|[ \t]*$/, "", line); |
|
return line; |
|
} |
|
|
|
function table_align(spec, a) { |
|
a = spec; |
|
gsub(/[ \t]/, "", a); |
|
if(a ~ /^:.*:$/) return "center"; |
|
if(a ~ /:$/) return "right"; |
|
if(a ~ /^:/) return "left"; |
|
return ""; |
|
} |
|
|
|
function table_row(line, celltag, n, i, cells, out, st) { |
|
n = split(table_cells(line), cells, "|"); |
|
out = "<tr>"; |
|
for(i = 1; i <= n; i++){ |
|
sub(/^[ \t]+/, "", cells[i]); |
|
sub(/[ \t]+$/, "", cells[i]); |
|
st = ""; |
|
if(tbl_align[i] != "") |
|
st = " style=\"text-align:" tbl_align[i] ";\""; |
|
out = out "\n\t<" celltag st ">" inline(cells[i]) "</" celltag ">"; |
|
} |
|
return out "\n</tr>"; |
|
} |
|
|
|
# A held header row, resolved by whatever follows it. |
|
tbl_pending != "" { |
|
if($0 ~ /^[ \t]*\|?[ \t]*:?-+:?[ \t]*(\|[ \t]*:?-+:?[ \t]*)*\|?[ \t]*$/){ |
|
# It was a table after all. |
|
n_align = split(table_cells($0), align_spec, "|"); |
|
for(ai = 1; ai <= n_align; ai++) |
|
tbl_align[ai] = table_align(align_spec[ai]); |
|
if(text) |
|
printp(par ? par : "p"); |
|
close_quotes(); |
|
oprint("<div class=\"table-scroll\">"); |
|
oprint("<table>"); |
|
oprint("<thead>"); |
|
oprint(table_row(tbl_pending, "th")); |
|
oprint("</thead>"); |
|
oprint("<tbody>"); |
|
tbl = 1; |
|
tbl_pending = ""; |
|
next; |
|
} |
|
# Not a table: the held line is ordinary text. |
|
text = (text ? text " " : "") tbl_pending; |
|
tbl_pending = ""; |
|
} |
|
|
|
tbl { |
|
if($0 ~ /^[ \t]*$/ || $0 !~ /\|/){ |
|
oprint("</tbody>"); |
|
oprint("</table>"); |
|
oprint("</div>"); |
|
tbl = 0; |
|
for(ci in tbl_align) delete tbl_align[ci]; |
|
if($0 ~ /^[ \t]*$/) |
|
next; |
|
} |
|
else{ |
|
oprint(table_row($0, "td")); |
|
next; |
|
} |
|
} |
|
|
|
!fence && !code && !tbl && /\|/ && !/^[ \t]*$/ && (!text || text == "") { |
|
# Candidate header row: hold it and let the next line decide. Only a line |
|
# whose pipes separate non-empty cells qualifies, so ordinary prose that |
|
# happens to contain a pipe is not swallowed. |
|
tbl_probe = $0; |
|
sub(/^[ \t]*\|/, "", tbl_probe); |
|
sub(/\|[ \t]*$/, "", tbl_probe); |
|
if(index(tbl_probe, "|") > 0 || $0 ~ /^[ \t]*\|.*\|[ \t]*$/){ |
|
tbl_pending = $0; |
|
next; |
|
} |
|
} |
|
|
|
# Fenced code blocks (MultiMarkdown / CommonMark) |
|
# |
|
# Must run before every other line rule: content inside a fence is literal, |
|
# so it has to bypass inline parsing, smart quotes and list detection alike. |
|
# The closing fence has to be at least as long as the opener and use the same |
|
# character, so a fence can contain a shorter run of backticks. |
|
fence { |
|
if($0 ~ fence_re && length($0) >= 0){ |
|
# Candidate closer: same char, at least as long, nothing after it. |
|
if(match($0, "^[ \t]*" fence_char "{" fence_len ",}[ \t]*$")){ |
|
if(fence_first) |
|
# An empty fence: nothing was ever buffered. |
|
oprint(fence_open fence_close); |
|
else |
|
oprint(fence_held fence_close); |
|
fence = 0; |
|
fence_held = ""; |
|
next; |
|
} |
|
} |
|
# Hold each line until the next one arrives, so the closing tag can be |
|
# joined to the last line rather than sitting on its own. |
|
if(fence_first){ |
|
fence_held = fence_open (fence_raw ? $0 : escfence($0)); |
|
fence_first = 0; |
|
} |
|
else{ |
|
oprint(fence_held); |
|
fence_held = (fence_raw ? $0 : escfence($0)); |
|
} |
|
next; |
|
} |
|
|
|
/^[ \t]*(```|~~~)/ { |
|
# Opening fence. Everything after the run is an info string; its first |
|
# word is the language. |
|
line = $0; |
|
sub(/^[ \t]*/, "", line); |
|
fence_char = substr(line, 1, 1); |
|
fence_len = 0; |
|
while(substr(line, fence_len + 1, 1) == fence_char) |
|
fence_len++; |
|
info = substr(line, fence_len + 1); |
|
sub(/^[ \t]+/, "", info); |
|
sub(/[ \t].*$/, "", info); |
|
|
|
# Close anything currently open so the block is not nested inside a <p>, |
|
# or inside a blockquote it merely follows. |
|
if(text) |
|
printp(par ? par : "p"); |
|
if(code){ |
|
oprint(code_held "</code></pre>"); code_held = ""; |
|
code = 0; |
|
} |
|
close_quotes(); |
|
|
|
fence = 1; |
|
fence_first = 1; |
|
fence_re = "^[ \t]*" fence_char; |
|
# A diagram language is held for a client-side renderer rather than shown |
|
# as source. The text goes in verbatim: mermaid reads the element's text |
|
# content and parses it itself, so escaping "-->" to "-->" would break |
|
# every arrow in the diagram. That makes this the one place a fence's |
|
# content is not escaped, which is safe only because the set of languages |
|
# treated this way is fixed here -- an author cannot introduce a new one |
|
# from a document. |
|
# |
|
# The language is kept as a class so the consumer decides what to do with |
|
# it. This parser renders no diagrams and loads nothing; a caller that |
|
# wants them supplies the renderer. |
|
fence_raw = 0; |
|
fence_close = "</code></pre>"; |
|
lang = info; |
|
sub(/[ \t].*$/, "", lang); |
|
if(lang == "mermaid" || lang == "mermaid-legend"){ |
|
fence_raw = 1; |
|
fence_close = "</div>"; |
|
fence_open = "<div class=\"diagram diagram-" escattr(lang) "\">"; |
|
} |
|
else if(info != "") |
|
fence_open = "<pre><code class=\"language-" escattr(info) "\">"; |
|
else |
|
fence_open = "<pre><code>"; |
|
next; |
|
} |
|
|
|
# List and quote blocks |
|
|
|
# Remove indentation |
|
# |
|
# One nesting level is peeled per step. A tab or four spaces was the only unit |
|
# understood here, but a child is conventionally indented to its parent's |
|
# content column, which for "- " is two spaces -- so two-space nesting, the |
|
# common form, was invisible: the item regex below tolerates up to three |
|
# leading spaces as noise, so a two-space child matched as a sibling and the |
|
# whole list came out flat. |
|
# |
|
# Two spaces is the step, not an alternative to four. Written as an |
|
# alternation with four spaces first, a four-space indent matched the wide |
|
# branch and peeled a single level, so three levels of four-space nesting |
|
# collapsed into two. A tab remains one whole level. |
|
{ |
|
for(nnl = 0; nnl < nl; nnl++) |
|
if((match(block[nnl + 1], /[ou]l/) && !sub(/^( | )/, "")) || \ |
|
(block[nnl + 1] == "blockquote" && !sub(/^> ?/, ""))) |
|
break; |
|
} |
|
nnl < nl && !blank && text && ! /^ ? ? ?([*+-]|([0-9]+\.)+)( +| )/ { nnl = nl; } |
|
# Quote blocks |
|
{ |
|
while(sub(/^> /, "")) |
|
nblock[++nnl] = "blockquote"; |
|
} |
|
# Horizontal rules |
|
{ hr = 0; } |
|
# A thematic break also ends a quoted paragraph that has no blank line after |
|
# it. The rule used to require !text, so after "> quote" the break was not |
|
# recognised at all and the dashes were swallowed into the paragraph as an |
|
# em-dash. A paragraph inside a quote is flushed first, then the break closes |
|
# the quote -- which is what CommonMark asks for: |
|
# |
|
# > quote <blockquote><p>quote</p></blockquote> |
|
# --- <hr> |
|
(blank || (!text && !code) || (text && block[nl] == "blockquote")) && \ |
|
/^ ? ? ?([-*_][ ]*)([-*_][ ]*)([-*_][ ]*)+$/ { |
|
if(code){ |
|
oprint(code_held "</code></pre>"); code_held = ""; |
|
code = 0; |
|
} |
|
if(text) |
|
printp(par ? par : "p"); |
|
blank = 0; |
|
nnl = 0; |
|
hr = 1; |
|
} |
|
# List items |
|
block[nl] ~ /[ou]l/ && /^$/ { |
|
blank = 1; |
|
next; |
|
} |
|
{ newli = 0; } |
|
!hr && (nnl != nl || !text || block[nl] ~ /[ou]l/) && /^ ? ? ?[*+-]( +| )/ { |
|
sub(/^ ? ? ?[*+-]( +| )/, ""); |
|
nnl++; |
|
nblock[nnl] = "ul"; |
|
newli = 1; |
|
} |
|
(nnl != nl || !text || block[nl] ~ /[ou]l/) && /^ ? ? ?([0-9]+\.)+( +| )/ { |
|
sub(/^ ? ? ?([0-9]+\.)+( +| )/, ""); |
|
nnl++; |
|
nblock[nnl] = "ol"; |
|
newli = 1; |
|
} |
|
newli { |
|
if(blank && nnl == nl && !par) |
|
par = "p"; |
|
blank = 0; |
|
printp(par); |
|
if(nnl == nl && block[nl] == nblock[nl]) |
|
oprint("</li><li>"); |
|
} |
|
blank && ! /^$/ { |
|
if(match(block[nnl], /[ou]l/) && !par) |
|
par = "p"; |
|
printp(par); |
|
par = "p"; |
|
blank = 0; |
|
} |
|
|
|
# Close old blocks and open new ones |
|
nnl != nl || nblock[nl] != block[nl] { |
|
if(code){ |
|
oprint(code_held "</code></pre>"); code_held = ""; |
|
code = 0; |
|
} |
|
printp(par); |
|
b = (nnl > nl) ? nblock[nnl] : block[nnl]; |
|
par = (match(b, /[ou]l/)) ? "" : "p"; |
|
} |
|
nnl < nl || (nnl == nl && nblock[nl] != block[nl]) { |
|
# The second clause compares against nblock, the block kind this line |
|
# wants, not pblock -- which is assigned nowhere and so was always empty, |
|
# making the test true for any open list. Once the walk reached nnl it |
|
# therefore kept going and closed the outer <ul> along with the nested one, |
|
# leaving a dedented sibling no list to rejoin. Harmless while nesting did |
|
# not work at all; load-bearing now that it does. |
|
for(; nl > nnl || (nl > 0 && nnl == nl && nblock[nl] != block[nl]); nl--){ |
|
if(match(block[nl], /[ou]l/)) |
|
oprint("</li>"); |
|
oprint("</" block[nl] ">"); |
|
} |
|
# A sibling after a nested list needs a fresh <li> in the list we just came |
|
# back to. The "</li><li>" rule earlier only fires when the item is already |
|
# at its own depth (nnl == nl), which is false on a dedent. |
|
if(newli && nnl > 0 && nnl == nl && match(block[nl], /[ou]l/) && block[nl] == nblock[nl]){ |
|
oprint("</li><li>"); |
|
newli = 0; |
|
} |
|
} |
|
nnl > nl { |
|
for(; nl < nnl; nl++){ |
|
block[nl + 1] = nblock[nl + 1]; |
|
oprint("<" block[nl + 1] ">"); |
|
if(match(block[nl + 1], /[ou]l/)) |
|
oprint("<li>"); |
|
} |
|
} |
|
hr { |
|
oprint("<hr>"); |
|
next; |
|
} |
|
|
|
# Code blocks |
|
code && /^$/ { |
|
if(blanK) |
|
oprint(""); |
|
blank = 1; |
|
next; |
|
} |
|
!text && sub(/^( | )/, "") { |
|
if(blanK) |
|
oprint(""); |
|
blank = 0; |
|
$0 = eschtml($0); |
|
if(!code){ |
|
code_held = "<pre><code>" $0; |
|
code = 1; |
|
} |
|
else{ |
|
oprint(code_held); |
|
code_held = $0; |
|
} |
|
next; |
|
} |
|
code { |
|
oprint(code_held "</code></pre>"); |
|
code_held = ""; |
|
code = 0; |
|
} |
|
|
|
# Setex-style Headers |
|
# |
|
# An underline belongs to the paragraph directly above it, and a paragraph |
|
# inside a blockquote is not directly above a line outside one. CommonMark is |
|
# explicit about the case: |
|
# |
|
# > foo |
|
# --- |
|
# |
|
# is a quoted paragraph followed by a thematic break, not a quoted heading. |
|
# The underline was reaching into the quote and turning "foo" into an <h2> |
|
# still inside it -- so the rule stops at the edge of a quote, and the line |
|
# falls through to the thematic-break handling that closes it. |
|
text && /^=+$/ && block[nl] != "blockquote" {printp("h1"); next;} |
|
text && /^-+$/ && block[nl] != "blockquote" {printp("h2"); next;} |
|
|
|
# Atx-Style headers |
|
/^#+/ && (!newli || par=="p" || /^##/) { |
|
for(n = 0; n < 6 && sub(/^# */, ""); n++) |
|
sub(/#$/, ""); |
|
par = "h" n; |
|
} |
|
|
|
# Paragraph |
|
/^$/ { |
|
printp(par); |
|
par = "p"; |
|
next; |
|
} |
|
|
|
# Add text |
|
{ text = (text ? text " " : "") $0; } |
|
|
|
function alen(a, ix, k) { |
|
k = 0 |
|
for(ix in a) k++ |
|
return k |
|
} |
|
|
|
END { |
|
# A candidate header row with no separator after it was ordinary text. |
|
if(tbl_pending != ""){ |
|
text = (text ? text " " : "") tbl_pending; |
|
tbl_pending = ""; |
|
} |
|
if(tbl){ |
|
oprint("</tbody>"); |
|
oprint("</table>"); |
|
oprint("</div>"); |
|
tbl = 0; |
|
} |
|
if(code){ |
|
oprint(code_held "</code></pre>"); code_held = ""; |
|
code = 0; |
|
} |
|
printp(par); |
|
for(; nl > 0; nl--){ |
|
if(match(block[nl], /[ou]l/)) |
|
oprint("</li>"); |
|
oprint("</" block[nl] ">"); |
|
} |
|
# A reference whose definition never appeared is not a link at all. |
|
# Restore the literal text the author wrote rather than emitting an |
|
# anchor with an empty href. |
|
for(sid in shortcut_ref){ |
|
lt = shortcut_ref[sid]; |
|
gsub("<a href=\"<<" sid "\">" lt "</a>", "[" lt "]", otext); |
|
} |
|
while(match(otext, /<a href="<<[^"]*">[^<]*<\/a>/)){ |
|
frag = substr(otext, RSTART, RLENGTH); |
|
lt = frag; |
|
sub(/^<a href="<<[^"]*">/, "", lt); |
|
sub(/<\/a>$/, "", lt); |
|
rid = frag; |
|
sub(/^<a href="<</, "", rid); |
|
sub(/">.*$/, "", rid); |
|
otext = substr(otext, 1, RSTART - 1) "[" lt "][" rid "]" \ |
|
substr(otext, RSTART + RLENGTH); |
|
} |
|
gsub(/<<[^"]*/, "", otext); |
|
print(otext); |
|
|
|
# Print citations, in order of first use. |
|
if(cite_n > 0) { |
|
print "<div class=\"citations\">"; |
|
print "<hr />"; |
|
print "<ol>"; |
|
for(ci = 1; ci <= cite_n; ci++){ |
|
cid = cite_order[ci]; |
|
print "<li id=\"cn:" escattr(ci) "\"><p>" inline(cite_body[cid]) \ |
|
" <a href=\"#cnref:" escattr(ci) "\" title=\"return to body\" class=\"reversecitation\"> ↩︎</a></p></li>"; |
|
} |
|
print "</ol>"; |
|
print "</div>"; |
|
} |
|
|
|
# Print footnotes |
|
if(alen(fnref)>0) { |
|
print "<ul class=\"fn-list\">"; |
|
for (i in fnref) print "<li id=\"fn-" escattr(i) "\" class=\"fn-item\"><span class=\"fn-handle\">" eschtml(i) ": </span><span class=\"fn-text\">" inline(fnref[i]) " <a href=\"#fnref-" escattr(i) "\" class=\"fn-backref\">↩︎</a></span></li>"; |
|
print "</ul>"; |
|
} |
|
} |