tools: overhaul tools/doc/html.js

PR-URL: https://github.com/nodejs/node/pull/20613
Reviewed-By: Trivikram Kamat <trivikr.dev@gmail.com>
This commit is contained in:
Vse Mozhet Byt 2018-04-25 18:24:27 +03:00
parent 64b50468bb
commit 12b0159adf

View File

@ -29,17 +29,10 @@ const typeParser = require('./type-parser.js');
module.exports = toHTML; module.exports = toHTML;
const STABILITY_TEXT_REG_EXP = /(.*:)\s*(\d)([\s\S]*)/; // Make `marked` to not automatically insert id attributes in headings.
const DOC_CREATED_REG_EXP = /<!--\s*introduced_in\s*=\s*v([0-9]+)\.([0-9]+)\.([0-9]+)\s*-->/;
// Customized heading without id attribute.
const renderer = new marked.Renderer(); const renderer = new marked.Renderer();
renderer.heading = function(text, level) { renderer.heading = (text, level) => `<h${level}>${text}</h${level}>\n`;
return `<h${level}>${text}</h${level}>\n`; marked.setOptions({ renderer });
};
marked.setOptions({
renderer: renderer
});
const docPath = path.resolve(__dirname, '..', '..', 'doc'); const docPath = path.resolve(__dirname, '..', '..', 'doc');
@ -47,91 +40,39 @@ const gtocPath = path.join(docPath, 'api', '_toc.md');
const gtocMD = fs.readFileSync(gtocPath, 'utf8').replace(/^@\/\/.*$/gm, ''); const gtocMD = fs.readFileSync(gtocPath, 'utf8').replace(/^@\/\/.*$/gm, '');
const gtocHTML = marked(gtocMD).replace( const gtocHTML = marked(gtocMD).replace(
/<a href="(.*?)"/g, /<a href="(.*?)"/g,
(all, href) => `<a class="nav-${toID(href)}" href="${href}"` (all, href) => `<a class="nav-${href.replace('.html', '')
.replace(/\W+/g, '-')}" href="${href}"`
); );
const templatePath = path.join(docPath, 'template.html'); const templatePath = path.join(docPath, 'template.html');
const template = fs.readFileSync(templatePath, 'utf8'); const template = fs.readFileSync(templatePath, 'utf8');
var docCreated = null; function toHTML({ input, filename, nodeVersion, analytics }, cb) {
var nodeVersion = null;
/**
* opts: input, filename, nodeVersion.
*/
function toHTML(opts, cb) {
nodeVersion = opts.nodeVersion || process.version;
docCreated = opts.input.match(DOC_CREATED_REG_EXP);
const lexed = marked.lexer(opts.input);
render({
lexed: lexed,
filename: opts.filename,
template: template,
nodeVersion: nodeVersion,
analytics: opts.analytics,
}, cb);
}
function toID(filename) {
return filename
.replace('.html', '')
.replace(/[^\w-]/g, '-')
.replace(/-+/g, '-');
}
/**
* opts: lexed, filename, template, nodeVersion.
*/
function render(opts, cb) {
var { lexed, filename, template } = opts;
const nodeVersion = opts.nodeVersion || process.version;
// Get the section.
const section = getSection(lexed);
filename = path.basename(filename, '.md'); filename = path.basename(filename, '.md');
parseText(lexed); const lexed = marked.lexer(input);
lexed = preprocessElements(lexed);
// Generate the table of contents. const firstHeading = lexed.find(({ type }) => type === 'heading');
// This mutates the lexed contents in-place. const section = firstHeading ? firstHeading.text : 'Index';
buildToc(lexed, filename, function(er, toc) {
if (er) return cb(er);
const id = toID(path.basename(filename)); preprocessText(lexed);
preprocessElements(lexed);
template = template.replace(/__ID__/g, id); // Generate the table of contents. This mutates the lexed contents in-place.
template = template.replace(/__FILENAME__/g, filename); const toc = buildToc(lexed, filename);
template = template.replace(/__SECTION__/g, section || 'Index');
template = template.replace(/__VERSION__/g, nodeVersion);
template = template.replace(/__TOC__/g, toc);
template = template.replace(
/__GTOC__/g,
gtocHTML.replace(`class="nav-${id}`, `class="nav-${id} active`)
);
if (opts.analytics) { const id = filename.replace(/\W+/g, '-');
template = template.replace(
'<!-- __TRACKING__ -->',
analyticsScript(opts.analytics)
);
}
template = template.replace(/__ALTDOCS__/, altDocs(filename)); let HTML = template.replace('__ID__', id)
.replace(/__FILENAME__/g, filename)
.replace('__SECTION__', section)
.replace(/__VERSION__/g, nodeVersion)
.replace('__TOC__', toc)
.replace('__GTOC__', gtocHTML.replace(
`class="nav-${id}`, `class="nav-${id} active`));
// Content has to be the last thing we do with the lexed tokens, if (analytics) {
// because it's destructive. HTML = HTML.replace('<!-- __TRACKING__ -->', `
const content = marked.parser(lexed);
template = template.replace(/__CONTENT__/g, content);
cb(null, template);
});
}
function analyticsScript(analytics) {
return `
<script src="assets/dnt_helper.js"></script> <script src="assets/dnt_helper.js"></script>
<script> <script>
if (!_dntEnabled()) { if (!_dntEnabled()) {
@ -143,149 +84,143 @@ function analyticsScript(analytics) {
ga('create', '${analytics}', 'auto'); ga('create', '${analytics}', 'auto');
ga('send', 'pageview'); ga('send', 'pageview');
} }
</script> </script>`);
`; }
}
// Replace placeholders in text tokens. const docCreated = input.match(
function replaceInText(text) { /<!--\s*introduced_in\s*=\s*v([0-9]+)\.([0-9]+)\.[0-9]+\s*-->/);
return linkJsTypeDocs(linkManPages(text)); if (docCreated) {
} HTML = HTML.replace('__ALTDOCS__', altDocs(filename, docCreated));
} else {
function altDocs(filename) {
if (!docCreated) {
console.error(`Failed to add alternative version links to ${filename}`); console.error(`Failed to add alternative version links to ${filename}`);
return ''; HTML = HTML.replace('__ALTDOCS__', '');
} }
function lte(v) { // Content insertion has to be the last thing we do with the lexed tokens,
const ns = v.num.split('.'); // because it's destructive.
if (docCreated[1] > +ns[0]) HTML = HTML.replace('__CONTENT__', marked.parser(lexed));
return false;
if (docCreated[1] < +ns[0])
return true;
return docCreated[2] <= +ns[1];
}
const versions = [ cb(null, HTML);
{ num: '10.x' },
{ num: '9.x' },
{ num: '8.x', lts: true },
{ num: '7.x' },
{ num: '6.x', lts: true },
{ num: '5.x' },
{ num: '4.x', lts: true },
{ num: '0.12.x' },
{ num: '0.10.x' }
];
const host = 'https://nodejs.org';
const href = (v) => `${host}/docs/latest-v${v.num}/api/${filename}.html`;
function li(v) {
let html = `<li><a href="${href(v)}">${v.num}`;
if (v.lts)
html += ' <b>LTS</b>';
return html + '</a></li>';
}
const lis = versions.filter(lte).map(li).join('\n');
if (!lis.length)
return '';
return `
<li class="version-picker">
<a href="#">View another version <span>&#x25bc;</span></a>
<ol class="version-picker">${lis}</ol>
</li>
`;
} }
// Handle general body-text replacements. // Handle general body-text replacements.
// For example, link man page references to the actual page. // For example, link man page references to the actual page.
function parseText(lexed) { function preprocessText(lexed) {
lexed.forEach(function(tok) { lexed.forEach((token) => {
if (tok.type === 'table') { if (token.type === 'table') {
if (tok.cells) { if (token.header) {
tok.cells.forEach((row, x) => { token.header = token.header.map(replaceInText);
row.forEach((_, y) => {
if (tok.cells[x] && tok.cells[x][y]) {
tok.cells[x][y] = replaceInText(tok.cells[x][y]);
}
});
});
} }
if (tok.header) { if (token.cells) {
tok.header.forEach((_, i) => { token.cells.forEach((row, i) => {
if (tok.header[i]) { token.cells[i] = row.map(replaceInText);
tok.header[i] = replaceInText(tok.header[i]);
}
}); });
} }
} else if (tok.text && tok.type !== 'code') { } else if (token.text && token.type !== 'code') {
tok.text = replaceInText(tok.text); token.text = replaceInText(token.text);
} }
}); });
} }
// Replace placeholders in text tokens.
function replaceInText(text) {
if (text === '') return text;
return linkJsTypeDocs(linkManPages(text));
}
// Syscalls which appear in the docs, but which only exist in BSD / macOS.
const BSD_ONLY_SYSCALLS = new Set(['lchmod']);
const MAN_PAGE = /(^|\s)([a-z.]+)\((\d)([a-z]?)\)/gm;
// Handle references to man pages, eg "open(2)" or "lchmod(2)".
// Returns modified text, with such refs replaced with HTML links, for example
// '<a href="http://man7.org/linux/man-pages/man2/open.2.html">open(2)</a>'.
function linkManPages(text) {
return text.replace(
MAN_PAGE, (match, beginning, name, number, optionalCharacter) => {
// Name consists of lowercase letters,
// number is a single digit with an optional lowercase letter.
const displayAs = `${name}(${number}${optionalCharacter})`;
if (BSD_ONLY_SYSCALLS.has(name)) {
return `${beginning}<a href="https://www.freebsd.org/cgi/man.cgi` +
`?query=${name}&sektion=${number}">${displayAs}</a>`;
}
return `${beginning}<a href="http://man7.org/linux/man-pages/man${number}` +
`/${name}.${number}${optionalCharacter}.html">${displayAs}</a>`;
});
}
const TYPE_SIGNATURE = /\{[^}]+\}/g;
function linkJsTypeDocs(text) {
const parts = text.split('`');
// Handle types, for example the source Markdown might say
// "This argument should be a {number} or {string}".
for (let i = 0; i < parts.length; i += 2) {
const typeMatches = parts[i].match(TYPE_SIGNATURE);
if (typeMatches) {
typeMatches.forEach((typeMatch) => {
parts[i] = parts[i].replace(typeMatch, typeParser.toLink(typeMatch));
});
}
}
return parts.join('`');
}
// Preprocess stability blockquotes and YAML blocks. // Preprocess stability blockquotes and YAML blocks.
function preprocessElements(input) { function preprocessElements(lexed) {
var state = null; const STABILITY_RE = /(.*:)\s*(\d)([\s\S]*)/;
const output = []; let state = null;
let headingIndex = -1; let headingIndex = -1;
let heading = null; let heading = null;
output.links = input.links; lexed.forEach((token, index) => {
input.forEach(function(tok, index) { if (token.type === 'heading') {
if (tok.type === 'heading') {
headingIndex = index; headingIndex = index;
heading = tok; heading = token;
} }
if (tok.type === 'html' && common.isYAMLBlock(tok.text)) { if (token.type === 'html' && common.isYAMLBlock(token.text)) {
tok.text = parseYAML(tok.text); token.text = parseYAML(token.text);
} }
if (tok.type === 'blockquote_start') { if (token.type === 'blockquote_start') {
state = 'MAYBE_STABILITY_BQ'; state = 'MAYBE_STABILITY_BQ';
return; lexed[index] = { type: 'space' };
} }
if (tok.type === 'blockquote_end' && state === 'MAYBE_STABILITY_BQ') { if (token.type === 'blockquote_end' && state === 'MAYBE_STABILITY_BQ') {
state = null; state = null;
return; lexed[index] = { type: 'space' };
} }
if (tok.type === 'paragraph' && state === 'MAYBE_STABILITY_BQ') { if (token.type === 'paragraph' && state === 'MAYBE_STABILITY_BQ') {
if (tok.text.match(/Stability:.*/g)) { if (token.text.includes('Stability:')) {
const stabilityMatch = tok.text.match(STABILITY_TEXT_REG_EXP); const [, prefix, number, explication] = token.text.match(STABILITY_RE);
const stability = Number(stabilityMatch[2]);
const isStabilityIndex = const isStabilityIndex =
index - 2 === headingIndex || // General. index - 2 === headingIndex || // General.
index - 3 === headingIndex; // With api_metadata block. index - 3 === headingIndex; // With api_metadata block.
if (heading && isStabilityIndex) { if (heading && isStabilityIndex) {
heading.stability = stability; heading.stability = number;
headingIndex = -1; headingIndex = -1;
heading = null; heading = null;
} }
tok.text = parseAPIHeader(tok.text).replace(/\n/g, ' '); token.text = `<div class="api_stability api_stability_${number}">` +
output.push({ type: 'html', text: tok.text }); '<a href="documentation.html#documentation_stability_index">' +
return; `${prefix} ${number}</a>${explication}</div>`
.replace(/\n/g, ' ');
lexed[index] = { type: 'html', text: token.text };
} else if (state === 'MAYBE_STABILITY_BQ') { } else if (state === 'MAYBE_STABILITY_BQ') {
output.push({ type: 'blockquote_start' });
state = null; state = null;
lexed[index - 1] = { type: 'blockquote_start' };
} }
} }
output.push(tok);
}); });
return output;
} }
function parseYAML(text) { function parseYAML(text) {
const meta = common.extractAndParseYAML(text); const meta = common.extractAndParseYAML(text);
const html = ['<div class="api_metadata">']; let html = '<div class="api_metadata">\n';
const added = { description: '' }; const added = { description: '' };
const deprecated = { description: '' }; const deprecated = { description: '' };
@ -302,153 +237,29 @@ function parseYAML(text) {
} }
if (meta.changes.length > 0) { if (meta.changes.length > 0) {
let changes = meta.changes.slice(); if (added.description) meta.changes.push(added);
if (added.description) changes.push(added); if (deprecated.description) meta.changes.push(deprecated);
if (deprecated.description) changes.push(deprecated);
changes = changes.sort((a, b) => versionSort(a.version, b.version)); meta.changes.sort((a, b) => versionSort(a.version, b.version));
html.push('<details class="changelog"><summary>History</summary>'); html += '<details class="changelog"><summary>History</summary>\n' +
html.push('<table>'); '<table>\n<tr><th>Version</th><th>Changes</th></tr>\n';
html.push('<tr><th>Version</th><th>Changes</th></tr>');
changes.forEach((change) => { meta.changes.forEach((change) => {
html.push(`<tr><td>${change.version}</td>`); html += `<tr><td>${change.version}</td>\n` +
html.push(`<td>${marked(change.description)}</td></tr>`); `<td>${marked(change.description)}</td></tr>\n`;
}); });
html.push('</table>'); html += '</table>\n</details>\n';
html.push('</details>');
} else { } else {
html.push(`${added.description}${deprecated.description}`); html += `${added.description}${deprecated.description}\n`;
} }
html.push('</div>'); html += '</div>';
return html.join('\n'); return html;
} }
// Syscalls which appear in the docs, but which only exist in BSD / macOS. const numberRe = /^\d*/;
const BSD_ONLY_SYSCALLS = new Set(['lchmod']);
// Handle references to man pages, eg "open(2)" or "lchmod(2)".
// Returns modified text, with such refs replaced with HTML links, for example
// '<a href="http://man7.org/linux/man-pages/man2/open.2.html">open(2)</a>'.
function linkManPages(text) {
return text.replace(
/(^|\s)([a-z.]+)\((\d)([a-z]?)\)/gm,
(match, beginning, name, number, optionalCharacter) => {
// Name consists of lowercase letters, number is a single digit.
const displayAs = `${name}(${number}${optionalCharacter})`;
if (BSD_ONLY_SYSCALLS.has(name)) {
return `${beginning}<a href="https://www.freebsd.org/cgi/man.cgi?query=${name}` +
`&sektion=${number}">${displayAs}</a>`;
} else {
return `${beginning}<a href="http://man7.org/linux/man-pages/man${number}` +
`/${name}.${number}${optionalCharacter}.html">${displayAs}</a>`;
}
});
}
function linkJsTypeDocs(text) {
const parts = text.split('`');
var i;
var typeMatches;
// Handle types, for example the source Markdown might say
// "This argument should be a {Number} or {String}".
for (i = 0; i < parts.length; i += 2) {
typeMatches = parts[i].match(/\{([^}]+)\}/g);
if (typeMatches) {
typeMatches.forEach(function(typeMatch) {
parts[i] = parts[i].replace(typeMatch, typeParser.toLink(typeMatch));
});
}
}
// TODO: maybe put more stuff here?
return parts.join('`');
}
function parseAPIHeader(text) {
const classNames = 'api_stability api_stability_$2';
const docsUrl = 'documentation.html#documentation_stability_index';
text = text.replace(
STABILITY_TEXT_REG_EXP,
`<div class="${classNames}"><a href="${docsUrl}">$1 $2</a>$3</div>`
);
return text;
}
// Section is just the first heading.
function getSection(lexed) {
for (var i = 0, l = lexed.length; i < l; i++) {
var tok = lexed[i];
if (tok.type === 'heading') return tok.text;
}
return '';
}
function getMark(anchor) {
return `<span><a class="mark" href="#${anchor}" id="${anchor}">#</a></span>`;
}
function buildToc(lexed, filename, cb) {
var toc = [];
var depth = 0;
const startIncludeRefRE = /^\s*<!-- \[start-include:(.+)\] -->\s*$/;
const endIncludeRefRE = /^\s*<!-- \[end-include:(.+)\] -->\s*$/;
const realFilenames = [filename];
lexed.forEach(function(tok) {
// Keep track of the current filename along @include directives.
if (tok.type === 'html') {
let match;
if ((match = tok.text.match(startIncludeRefRE)) !== null)
realFilenames.unshift(match[1]);
else if (tok.text.match(endIncludeRefRE))
realFilenames.shift();
}
if (tok.type !== 'heading') return;
if (tok.depth - depth > 1) {
return cb(new Error('Inappropriate heading level\n' +
JSON.stringify(tok)));
}
depth = tok.depth;
const realFilename = path.basename(realFilenames[0], '.md');
const apiName = tok.text.trim();
const id = getId(`${realFilename}_${apiName}`);
toc.push(new Array((depth - 1) * 2 + 1).join(' ') +
`* <span class="stability_${tok.stability}">` +
`<a href="#${id}">${tok.text}</a></span>`);
tok.text += getMark(id);
if (realFilename === 'errors' && apiName.startsWith('ERR_')) {
tok.text += getMark(apiName);
}
});
toc = marked.parse(toc.join('\n'));
cb(null, toc);
}
const idCounters = {};
function getId(text) {
text = text.toLowerCase();
text = text.replace(/[^a-z0-9]+/g, '_');
text = text.replace(/^_+|_+$/, '');
text = text.replace(/^([^a-z])/, '_$1');
if (idCounters.hasOwnProperty(text)) {
text += `_${++idCounters[text]}`;
} else {
idCounters[text] = 0;
}
return text;
}
const numberRe = /^(\d*)/;
function versionSort(a, b) { function versionSort(a, b) {
a = a.trim(); a = a.trim();
b = b.trim(); b = b.trim();
@ -456,5 +267,100 @@ function versionSort(a, b) {
while (i < a.length && i < b.length && a[i] === b[i]) i++; while (i < a.length && i < b.length && a[i] === b[i]) i++;
a = a.substr(i); a = a.substr(i);
b = b.substr(i); b = b.substr(i);
return +b.match(numberRe)[1] - +a.match(numberRe)[1]; return +b.match(numberRe)[0] - +a.match(numberRe)[0];
}
function buildToc(lexed, filename) {
const startIncludeRefRE = /^\s*<!-- \[start-include:(.+)\] -->\s*$/;
const endIncludeRefRE = /^\s*<!-- \[end-include:.+\] -->\s*$/;
const realFilenames = [filename];
const idCounters = Object.create(null);
let toc = '';
let depth = 0;
lexed.forEach((token) => {
// Keep track of the current filename along comment wrappers of inclusions.
if (token.type === 'html') {
const [, includedFileName] = token.text.match(startIncludeRefRE) || [];
if (includedFileName !== undefined)
realFilenames.unshift(includedFileName);
else if (endIncludeRefRE.test(token.text))
realFilenames.shift();
}
if (token.type !== 'heading') return;
if (token.depth - depth > 1) {
throw new Error(`Inappropriate heading level:\n${JSON.stringify(token)}`);
}
depth = token.depth;
const realFilename = path.basename(realFilenames[0], '.md');
const headingText = token.text.trim();
const id = getId(`${realFilename}_${headingText}`, idCounters);
toc += ' '.repeat((depth - 1) * 2) +
`* <span class="stability_${token.stability}">` +
`<a href="#${id}">${token.text}</a></span>\n`;
token.text += `<span><a class="mark" href="#${id}" id="${id}">#</a></span>`;
if (realFilename === 'errors' && headingText.startsWith('ERR_')) {
token.text += `<span><a class="mark" href="#${headingText}" ` +
`id="${headingText}">#</a></span>`;
}
});
return marked(toc);
}
const notAlphaNumerics = /[^a-z0-9]+/g;
const edgeUnderscores = /^_+|_+$/g;
const notAlphaStart = /^[^a-z]/;
function getId(text, idCounters) {
text = text.toLowerCase()
.replace(notAlphaNumerics, '_')
.replace(edgeUnderscores, '')
.replace(notAlphaStart, '_$&');
if (idCounters[text] !== undefined) {
return `${text}_${++idCounters[text]}`;
}
idCounters[text] = 0;
return text;
}
function altDocs(filename, docCreated) {
const [, docCreatedMajor, docCreatedMinor] = docCreated.map(Number);
const host = 'https://nodejs.org';
const versions = [
{ num: '10.x' },
{ num: '9.x' },
{ num: '8.x', lts: true },
{ num: '7.x' },
{ num: '6.x', lts: true },
{ num: '5.x' },
{ num: '4.x', lts: true },
{ num: '0.12.x' },
{ num: '0.10.x' }
];
const getHref = (versionNum) =>
`${host}/docs/latest-v${versionNum}/api/${filename}.html`;
const wrapInListItem = (version) =>
`<li><a href="${getHref(version.num)}">${version.num}` +
`${version.lts ? ' <b>LTS</b>' : ''}</a></li>`;
function isDocInVersion(version) {
const [versionMajor, versionMinor] = version.num.split('.').map(Number);
if (docCreatedMajor > versionMajor) return false;
if (docCreatedMajor < versionMajor) return true;
return docCreatedMinor <= versionMinor;
}
const list = versions.filter(isDocInVersion).map(wrapInListItem).join('\n');
return list ? `
<li class="version-picker">
<a href="#">View another version <span>&#x25bc;</span></a>
<ol class="version-picker">${list}</ol>
</li>
` : '';
} }