Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 15 additions & 33 deletions src/Page.js
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@ const fs = require('fs-extra-promise');
const htmlBeautify = require('js-beautify').html;
const nunjucks = require('nunjucks');
const path = require('path');
const pathIsInside = require('path-is-inside');
const Promise = require('bluebird');
const nunjuckUtils = require('./lib/markbind/src/utils/nunjuckUtils');

Expand All @@ -20,6 +19,7 @@ const logger = require('./util/logger');
const MarkBind = require('./lib/markbind/src/parser');
const md = require('./lib/markbind/src/lib/markdown-it');
const utils = require('./lib/markbind/src/utils');
const urlUtils = require('./lib/markbind/src/utils/urls');

const CLI_VERSION = require('../package.json').version;

Expand Down Expand Up @@ -480,8 +480,8 @@ class Page {
// Set header file as an includedFile
this.includedFiles.add(headerPath);
// Map variables
const newBaseUrl = Page.calculateNewBaseUrl(this.sourcePath, this.rootPath, this.baseUrlMap) || '';
const userDefinedVariables = this.userDefinedVariablesMap[path.join(this.rootPath, newBaseUrl)];
const parentSite = urlUtils.getParentSiteAbsolutePath(this.sourcePath, this.rootPath, this.baseUrlMap);
const userDefinedVariables = this.userDefinedVariablesMap[parentSite];
return `${nunjuckUtils.renderEscaped(nunjucks, headerContent, userDefinedVariables)}\n${pageData}`;
}

Expand Down Expand Up @@ -510,8 +510,8 @@ class Page {
// Set footer file as an includedFile
this.includedFiles.add(footerPath);
// Map variables
const newBaseUrl = Page.calculateNewBaseUrl(this.sourcePath, this.rootPath, this.baseUrlMap) || '';
const userDefinedVariables = this.userDefinedVariablesMap[path.join(this.rootPath, newBaseUrl)];
const parentSite = urlUtils.getParentSiteAbsolutePath(this.sourcePath, this.rootPath, this.baseUrlMap);
const userDefinedVariables = this.userDefinedVariablesMap[parentSite];
return `${pageData}\n${nunjuckUtils.renderEscaped(nunjucks, footerContent, userDefinedVariables)}`;
}

Expand Down Expand Up @@ -547,8 +547,8 @@ class Page {
// Set siteNav file as an includedFile
this.includedFiles.add(siteNavPath);
// Map variables
const newBaseUrl = Page.calculateNewBaseUrl(this.sourcePath, this.rootPath, this.baseUrlMap) || '';
const userDefinedVariables = this.userDefinedVariablesMap[path.join(this.rootPath, newBaseUrl)];
const parentSite = urlUtils.getParentSiteAbsolutePath(this.sourcePath, this.rootPath, this.baseUrlMap);
const userDefinedVariables = this.userDefinedVariablesMap[parentSite];
const siteNavMappedData = nunjuckUtils.renderEscaped(nunjucks, siteNavContent, userDefinedVariables);
// Convert to HTML
const siteNavDataSelector = cheerio.load(siteNavMappedData);
Expand Down Expand Up @@ -697,8 +697,8 @@ class Page {
// Set head file as an includedFile
this.includedFiles.add(headFilePath);
// Map variables
const newBaseUrl = Page.calculateNewBaseUrl(this.sourcePath, this.rootPath, this.baseUrlMap) || '';
const userDefinedVariables = this.userDefinedVariablesMap[path.join(this.rootPath, newBaseUrl)];
const parentSite = urlUtils.getParentSiteAbsolutePath(this.sourcePath, this.rootPath, this.baseUrlMap);
const userDefinedVariables = this.userDefinedVariablesMap[parentSite];
const headFileMappedData = nunjuckUtils.renderEscaped(nunjucks, headFileContent, userDefinedVariables)
.trim();
// Split top and bottom contents
Expand Down Expand Up @@ -808,8 +808,9 @@ class Page {
.then((result) => {
this.content = htmlBeautify(result, Page.htmlBeautifyOptions);

const newBaseUrl = Page.calculateNewBaseUrl(this.sourcePath, this.rootPath, this.baseUrlMap);
const baseUrl = newBaseUrl ? `${this.baseUrl}/${newBaseUrl}` : this.baseUrl;
const { relative } = urlUtils.getParentSiteAbsoluteAndRelativePaths(this.sourcePath, this.rootPath,
this.baseUrlMap);
const baseUrl = relative ? `${this.baseUrl}/${utils.ensurePosix(relative)}` : this.baseUrl;
const hostBaseUrl = this.baseUrl;

this.addLayoutFiles();
Expand Down Expand Up @@ -1026,7 +1027,6 @@ class Page {
* @param builtFiles set of files already pre-rendered by another page
*/
resolveDependency(dependency, builtFiles) {
const source = dependency.from;
const file = dependency.asIfTo;
return new Promise((resolve, reject) => {
const resultDir = path.dirname(path.resolve(this.resultPath, path.relative(this.sourcePath, file)));
Expand Down Expand Up @@ -1061,8 +1061,6 @@ class Page {
.then(result => markbinder.resolveBaseUrl(result, {
baseUrlMap: this.baseUrlMap,
rootPath: this.rootPath,
isDynamic: true,
Copy link
Contributor Author

@ang-zeyu ang-zeyu Mar 4, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

these variables in resolveDependency are not necessary, as in the resolveBaseUrl process, attribs[ATTRIB_CWF] = context.cwf is the same as dependency.from ( ie. parser.dynamicIncludeSrc.push({ from: context.cwf, ... }) )

dynamicSource: source,
}))
.then(result => fs.outputFileAsync(tempPath, result))
.then(() => markbinder.renderFile(tempPath, {
Expand All @@ -1075,8 +1073,9 @@ class Page {
.then(result => markbinder.processDynamicResources(file, result))
.then((result) => {
// resolve the site base url here
const newBaseUrl = Page.calculateNewBaseUrl(file, this.rootPath, this.baseUrlMap);
const baseUrl = newBaseUrl ? `${this.baseUrl}/${newBaseUrl}` : this.baseUrl;
const { relative } = urlUtils.getParentSiteAbsoluteAndRelativePaths(file, this.rootPath,
this.baseUrlMap);
const baseUrl = relative ? `${this.baseUrl}/${utils.ensurePosix(relative)}` : this.baseUrl;
const hostBaseUrl = this.baseUrl;
const content = nunjucks.renderString(result, {
baseUrl,
Expand Down Expand Up @@ -1115,23 +1114,6 @@ class Page {
+ '</div>';
}

static calculateNewBaseUrl(filePath, root, lookUp) {
function calculate(file, result) {
if (file === root || !pathIsInside(file, root)) {
return undefined;
}
const parent = path.dirname(file);
if (lookUp.has(parent) && result.length === 1) {
return path.relative(root, result[0]);
} else if (lookUp.has(parent)) {
return calculate(parent, [parent]);
}
return calculate(parent, result);
}

return calculate(filePath, []);
}

static removePageHeaderAndFooter(pageData) {
const $ = cheerio.load(pageData);
const pageHeaderAndFooter = $('header', 'footer');
Expand Down
1 change: 1 addition & 0 deletions src/lib/markbind/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
"@sindresorhus/slugify": "^0.9.1",
"bluebird": "^3.7.2",
"cheerio": "^0.22.0",
"ensure-posix-path": "^1.1.1",
"fastmatter": "^2.1.1",
"highlight.js": "^9.14.2",
"htmlparser2": "^3.10.1",
Expand Down
114 changes: 49 additions & 65 deletions src/lib/markbind/src/parser.js
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ const nunjucks = require('nunjucks');
const path = require('path');
const Promise = require('bluebird');
const slugify = require('@sindresorhus/slugify');
const ensurePosix = require('ensure-posix-path');
const componentParser = require('./parsers/componentParser');
const componentPreprocessor = require('./preprocessors/componentPreprocessor');
const nunjuckUtils = require('./utils/nunjuckUtils');
Expand Down Expand Up @@ -139,8 +140,8 @@ class Parser {
return utils.createErrorNode(node, e);
}
const fileContent = this._fileCache[filePath]; // cache the file contents to save some I/O
const { parent, relative } = urlUtils.calculateNewBaseUrls(asIfAt, config.rootPath, config.baseUrlMap);
const userDefinedVariables = config.userDefinedVariablesMap[path.resolve(parent, relative)];
const parentSitePath = urlUtils.getParentSiteAbsolutePath(asIfAt, config.rootPath, config.baseUrlMap);
const userDefinedVariables = config.userDefinedVariablesMap[parentSitePath];
// Extract included variables from the PARENT file
const includeVariables = Parser.extractIncludeVariables(node, context.variables);
// Extract page variables from the CHILD file
Expand Down Expand Up @@ -395,9 +396,8 @@ class Parser {
reject(err);
return;
}
const { parent, relative }
= urlUtils.calculateNewBaseUrls(file, config.rootPath, config.baseUrlMap);
const userDefinedVariables = config.userDefinedVariablesMap[path.resolve(parent, relative)];
const parentSitePath = urlUtils.getParentSiteAbsolutePath(file, config.rootPath, config.baseUrlMap);
const userDefinedVariables = config.userDefinedVariablesMap[parentSitePath];
const pageVariables = this.extractPageVariables(file, data, userDefinedVariables, {});
let fileContent = nunjuckUtils.renderEscaped(nunjucks, data, {
...pageVariables,
Expand Down Expand Up @@ -466,9 +466,8 @@ class Parser {
decodeEntities: true,
});

const { parent, relative } = urlUtils.calculateNewBaseUrls(actualFilePath,
config.rootPath, config.baseUrlMap);
const userDefinedVariables = config.userDefinedVariablesMap[path.resolve(parent, relative)];
const parentSitePath = urlUtils.getParentSiteAbsolutePath(file, config.rootPath, config.baseUrlMap);
const userDefinedVariables = config.userDefinedVariablesMap[parentSitePath];
const { additionalVariables } = config;
const pageVariables = this.extractPageVariables(actualFilePath, pageData, userDefinedVariables, {});

Expand Down Expand Up @@ -555,30 +554,17 @@ class Parser {
}

resolveBaseUrl(pageData, config) {
const { baseUrlMap, rootPath, isDynamic } = config;
const { baseUrlMap, rootPath } = config;
this.baseUrlMap = baseUrlMap;
this.rootPath = rootPath;
this.isDynamic = isDynamic || false;
if (this.isDynamic) {
this.dynamicSource = config.dynamicSource;
}

return new Promise((resolve, reject) => {
const handler = new htmlparser.DomHandler((error, dom) => {
if (error) {
reject(error);
return;
}
const nodes = dom.map((d) => {
const node = d;
const childrenBase = {};
if (this.isDynamic) {
// Change CWF for each top level element
if (node.attribs) {
node.attribs[ATTRIB_CWF] = this.dynamicSource;
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

see the comment for resolveDependency

}
}
return this._rebaseReference(node, childrenBase);
});
const nodes = dom.map(d => this._rebaseReference(d));
cheerio.prototype.options.xmlMode = false;
resolve(cheerio.html(nodes));
cheerio.prototype.options.xmlMode = true;
Expand All @@ -591,53 +577,51 @@ class Parser {
});
}

_rebaseReference(node, foundBase) {
/**
* Pre-renders the baseUrl of the provided node and its children according to
* the site they belong to, such that the final call to render baseUrl correctly
* resolves baseUrl according to the said site.
*/
_rebaseReference(node) {
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Two changes here essentially:

  • replace nested ifs with guard clauses
  • removal of foundBase ( see the explanation in the OP )

if (_.isArray(node)) {
return node.map(el => this._rebaseReference(el, foundBase));
return node.map(el => this._rebaseReference(el));
}
if (Parser.isText(node)) {
return node;
}
// Rebase children element
const childrenBase = {};
node.children.forEach((el) => {
this._rebaseReference(el, childrenBase);
});
// rebase current element
if (node.attribs[ATTRIB_INCLUDE_PATH]) {
const filePath = node.attribs[ATTRIB_INCLUDE_PATH];
let newBaseUrl = urlUtils.calculateNewBaseUrls(filePath, this.rootPath, this.baseUrlMap);
if (newBaseUrl) {
const { relative, parent } = newBaseUrl;
// eslint-disable-next-line no-param-reassign
foundBase[parent] = relative;
}
// override with parent's base
const combinedBases = { ...childrenBase, ...foundBase };
Copy link
Contributor

@marvinchin marvinchin Mar 8, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This seems to be the PR that added the combinedBases change:
#263

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks for this! Unfortunately the PR dosen't really reveal the workings of combinedBases, but

What is the rationale for this request?
When including files belonging to an inner website, the computation of baseUrl is wrong if the depth of the inclusion is only 1.

Going by this, it should be sufficient to resolve the baseurl only when the include's file and included file belongs to different sites. The likely cause of the issue was wrongly resolving against ATTRIB_CWF instead of ATTRIB_INCLUDE_PATH, and perhaps the above mentioned irregularities with foundBase.

I was looking at 64b2ff6 previously, where foundBase was included with the function right from the start.
It dosen't reveal much as well unfortunately, perhaps @Gisonrg could comment!

const bases = Object.keys(combinedBases);
if (bases.length !== 0) {
// need to rebase
newBaseUrl = combinedBases[bases[0]];
if (node.children) {
// ATTRIB_CWF is where the element was preprocessed
const currentBase = urlUtils.calculateNewBaseUrls(node.attribs[ATTRIB_CWF],
this.rootPath, this.baseUrlMap);
if (currentBase && currentBase.relative !== newBaseUrl) {
cheerio.prototype.options.xmlMode = false;
const rendered = nunjuckUtils.renderEscaped(nunjucks, cheerio.html(node.children), {
// This is to prevent the nunjuck call from converting {{hostBaseUrl}} to an empty string
// and let the hostBaseUrl value be injected later.
hostBaseUrl: '{{hostBaseUrl}}',
baseUrl: `{{hostBaseUrl}}/${newBaseUrl}`,
}, { path: filePath });
node.children = cheerio.parseHTML(rendered, true);
cheerio.prototype.options.xmlMode = true;
}
}
}
delete node.attribs[ATTRIB_INCLUDE_PATH];
}

// Rebase children elements
node.children.forEach(el => this._rebaseReference(el));

const includeSourceFile = node.attribs[ATTRIB_CWF];
const includedSourceFile = node.attribs[ATTRIB_INCLUDE_PATH];
delete node.attribs[ATTRIB_CWF];
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Just wondering - should we delete these attributes here? It feels a little odd since they were added in _preprocess (at least, I think), and it isn't immediately intuitive that these attributes would be deleted in this function.

If it is necessary for us to delete it here, could we leave a comment explaining why we need to do this? And maybe add a comment in _preprocess to notify that this variable will be deleted here.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The original function already deletes both

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It's not necessary to the best of my knowledge, but not having <div cwf="..."> all over the output is a nice touch

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ah noted I didn't notice that these outputs would be included in the output!

delete node.attribs[ATTRIB_INCLUDE_PATH];

// Skip rebasing for non-includes
if (!includedSourceFile || !node.children) {
return node;
}

// The site at which the file with the include element was pre-processed
const currentBase = urlUtils.getParentSiteAbsolutePath(includeSourceFile, this.rootPath, this.baseUrlMap);
// The site of the included file
const newBase = urlUtils.getParentSiteAbsoluteAndRelativePaths(includedSourceFile, this.rootPath,
this.baseUrlMap);

// Only re-render if include src and content are from different sites
if (currentBase !== newBase.absolute) {
cheerio.prototype.options.xmlMode = false;
const rendered = nunjuckUtils.renderEscaped(nunjucks, cheerio.html(node.children), {
// This is to prevent the nunjuck call from converting {{hostBaseUrl}} to an empty string
// and let the hostBaseUrl value be injected later.
hostBaseUrl: '{{hostBaseUrl}}',
baseUrl: `{{hostBaseUrl}}/${ensurePosix(newBase.relative)}`,
}, { path: includedSourceFile });
node.children = cheerio.parseHTML(rendered, true);
cheerio.prototype.options.xmlMode = true;
}

return node;
}

Expand Down
23 changes: 0 additions & 23 deletions src/lib/markbind/src/preprocessors/componentPreprocessor.js
Original file line number Diff line number Diff line change
Expand Up @@ -182,28 +182,6 @@ function _preProcessPanel(node, context, config, parser) {
* Includes
*/


function _rebaseReferenceForStaticIncludes(pageData, element, config) {
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

See #1083

if (!config) return pageData;

if (!pageData.includes('{{baseUrl}}')) return pageData;

const filePath = element.attribs[ATTRIB_INCLUDE_PATH];
const fileBase = urlUtils.calculateNewBaseUrls(filePath, config.rootPath, config.baseUrlMap);

if (!fileBase.relative) return pageData;

const currentPath = element.attribs[ATTRIB_CWF];
const currentBase = urlUtils.calculateNewBaseUrls(currentPath, config.rootPath, config.baseUrlMap);

if (currentBase.relative === fileBase.relative) return pageData;

const newBase = fileBase.relative;
const newBaseUrl = `{{hostBaseUrl}}/${newBase}`;

return nunjuckUtils.renderEscaped(nunjucks, pageData, { baseUrl: newBaseUrl }, { path: filePath });
}

function _deleteIncludeAttributes(node) {
const element = node;

Expand Down Expand Up @@ -334,7 +312,6 @@ function _preprocessInclude(node, context, config, parser) {
if (isIncludeSrcMd) {
actualContent = isInline ? actualContent : `\n\n${actualContent}\n`;
}
actualContent = _rebaseReferenceForStaticIncludes(actualContent, element, config);

// Flag with a data-included-from flag with the source filePath for calculating
// the file path of dynamic resources ( images, anchors, plugin sources, etc. ) later
Expand Down
Loading