', 'specify the site config file (default: site.json)')
.action((userSpecifiedRoot, options) => {
let rootFolder;
try {
rootFolder = cliUtil.findRootFolder(userSpecifiedRoot, options.siteConfig);
+
+ if (options.forceReload && options.onePage) {
+ handleError(new Error('Oops! You shouldn\'t need to use the --force-reload option with --one-page.'));
+ process.exit();
+ }
} catch (err) {
handleError(err);
}
const logsFolder = path.join(rootFolder, '_markbind/logs');
const outputFolder = path.join(rootFolder, '_site');
- if (options.onePage) {
- // replace slashes for paths on Windows
- // eslint-disable-next-line no-param-reassign
- options.onePage = ensurePosix(options.onePage);
- }
+ let onePagePath = options.onePage === true ? INDEX_MARKDOWN_FILE : options.onePage;
+ onePagePath = onePagePath ? ensurePosix(onePagePath) : onePagePath;
- const site = new Site(rootFolder, outputFolder, options.onePage, options.forceReload, options.siteConfig);
+ const site = new Site(rootFolder, outputFolder, onePagePath, options.forceReload, options.siteConfig);
const addHandler = (filePath) => {
logger.info(`[${new Date().toLocaleTimeString()}] Reload for file add: ${filePath}`);
@@ -152,12 +158,15 @@ program
});
};
+ const onePageHtmlUrl = onePagePath && `/${onePagePath.replace(/\.(md|mbd|mbdf)$/, '.html')}`;
+
// server config
const serverConfig = {
- open: options.open && (options.onePage ? `/${options.onePage.replace(/\.(md|mbd)$/, '.html')}` : true),
+ open: options.open && (onePageHtmlUrl || true),
logLevel: 0,
root: outputFolder,
port: options.port || 8080,
+ middleware: [],
mount: [],
};
@@ -167,6 +176,30 @@ program
.readSiteConfig()
.then((config) => {
serverConfig.mount.push([config.baseUrl || '/', outputFolder]);
+
+ if (onePagePath) {
+ const lazyReloadMiddleware = function (req, res, next) {
+ const isHtmlFile = req.url.endsWith('.html');
+
+ if (isHtmlFile) {
+ const isDynamicIncludeHtmlFile = req.url.endsWith('._include_.html');
+
+ if (!isDynamicIncludeHtmlFile) {
+ const urlWithoutBaseUrl = req.url.replace(config.baseUrl, '');
+ const urlWithoutExtension = fsUtil.removeExtension(urlWithoutBaseUrl);
+
+ const didInitiateRebuild = site.changeCurrentPage(urlWithoutExtension);
+ if (didInitiateRebuild) {
+ req.url = ensurePosix(path.join(config.baseUrl || '/', LAZY_LOADING_SITE_FILE_NAME));
+ }
+ }
+ }
+ next();
+ };
+
+ serverConfig.middleware.push(lazyReloadMiddleware);
+ }
+
return site.generate();
})
.then(() => {
diff --git a/src/LazyLiveReloadLoadingSite.html b/src/LazyLiveReloadLoadingSite.html
new file mode 100644
index 0000000000..834773557c
--- /dev/null
+++ b/src/LazyLiveReloadLoadingSite.html
@@ -0,0 +1,66 @@
+
+
+
+
+ Building ...
+
+
+
+ ... building your page ...
+
+
+
+
diff --git a/src/Site.js b/src/Site.js
index c7b4afca69..916cf0b9d7 100644
--- a/src/Site.js
+++ b/src/Site.js
@@ -48,6 +48,9 @@ const {
LAYOUT_DEFAULT_NAME,
LAYOUT_FOLDER_PATH,
LAYOUT_SITE_FOLDER_NAME,
+ LAZY_LOADING_SITE_FILE_NAME,
+ LAZY_LOADING_BUILD_TIME_RECOMMENDATION_LIMIT,
+ LAZY_LOADING_REBUILD_TIME_RECOMMENDATION_LIMIT,
MARKBIND_PLUGIN_PREFIX,
MARKBIND_WEBSITE_URL,
PAGE_TEMPLATE_NAME,
@@ -124,11 +127,17 @@ class Site {
this.addressablePages = [];
this.baseUrlMap = new Set();
this.forceReload = forceReload;
- this.onePagePath = onePagePath;
this.plugins = {};
this.siteConfig = {};
this.siteConfigPath = siteConfigPath;
this.userDefinedVariablesMap = {};
+
+ // Lazy reload properties
+ this.onePagePath = onePagePath;
+ this.currentPageViewed = onePagePath
+ ? path.resolve(this.rootPath, FsUtil.removeExtension(onePagePath))
+ : '';
+ this.toRebuild = new Set();
}
/**
@@ -170,6 +179,22 @@ class Site {
});
}
+ /**
+ * Changes the site variable of the current page being viewed, building it if necessary.
+ * @param normalizedUrl BaseUrl-less and extension-less url of the page
+ * @return Boolean of whether the page needed to be rebuilt
+ */
+ changeCurrentPage(normalizedUrl) {
+ this.currentPageViewed = path.join(this.rootPath, normalizedUrl);
+
+ if (this.toRebuild.has(this.currentPageViewed)) {
+ this.rebuildPageBeingViewed(this.currentPageViewed);
+ return true;
+ }
+
+ return false;
+ }
+
readSiteConfig(baseUrl) {
return new Promise((resolve, reject) => {
const siteConfigPath = path.join(this.rootPath, this.siteConfigPath);
@@ -405,10 +430,11 @@ class Site {
* Updates the paths to be traversed as addressable pages and returns a list of filepaths to be deleted
*/
updateAddressablePages() {
- const oldAddressablePages = this.addressablePages.slice();
+ const oldAddressablePagesSources = this.addressablePages.slice().map(page => page.src);
this.collectAddressablePages();
- return _.difference(oldAddressablePages.map(page => page.src),
- this.addressablePages.map(page => page.src))
+ const newAddressablePagesSources = this.addressablePages.map(page => page.src);
+
+ return _.difference(oldAddressablePagesSources, newAddressablePagesSources)
.map(filePath => Site.setExtension(filePath, '.html'));
}
@@ -521,7 +547,9 @@ class Site {
fs.emptydirSync(this.tempPath);
// Clean the output folder; create it if not exist.
fs.emptydirSync(this.outputPath);
- logger.info(`Website generation started at ${startTime.toLocaleTimeString()}`);
+ const lazyWebsiteGenerationString = this.onePagePath ? '(lazy) ' : '';
+ logger.info(`Website generation ${lazyWebsiteGenerationString}started at ${
+ startTime.toLocaleTimeString()}`);
return new Promise((resolve, reject) => {
this.readSiteConfig(baseUrl)
.then(() => this.collectAddressablePages())
@@ -530,16 +558,22 @@ class Site {
.then(() => this.collectPlugins())
.then(() => this.collectPluginSpecialTags())
.then(() => this.buildAssets())
- .then(() => this.buildSourceFiles())
+ .then(() => (this.onePagePath ? this.lazyBuildSourceFiles() : this.buildSourceFiles()))
.then(() => this.copyMarkBindAsset())
.then(() => this.copyFontAwesomeAsset())
.then(() => this.copyOcticonsAsset())
.then(() => this.copyLayouts())
- .then(() => this.updateSiteData())
+ .then(() => this.updateSiteData(this.onePagePath || undefined))
.then(() => {
const endTime = new Date();
const totalBuildTime = (endTime - startTime) / 1000;
- logger.info(`Website generation complete! Total build time: ${totalBuildTime}s`);
+ logger.info(`Website generation ${lazyWebsiteGenerationString}complete! Total build time: ${
+ totalBuildTime}s`);
+
+ if (!this.onePagePath && totalBuildTime > LAZY_LOADING_BUILD_TIME_RECOMMENDATION_LIMIT) {
+ logger.info('Your site took quite a while to build...'
+ + 'Have you considered using markbind serve -o when writing content to speed things up?');
+ }
})
.then(resolve)
.catch((error) => {
@@ -565,6 +599,44 @@ class Site {
});
}
+ /**
+ * Adds all pages except the current page being viewed to toRebuild, flagging them for lazy building later.
+ */
+ lazyBuildAllPagesNotViewed() {
+ this.pages.forEach((page) => {
+ const normalizedUrl = FsUtil.removeExtension(page.sourcePath);
+ if (normalizedUrl !== this.currentPageViewed) {
+ this.toRebuild.add(normalizedUrl);
+ }
+ });
+
+ return Promise.resolve();
+ }
+
+ /**
+ * Only build landing page of the site, building more as the author goes to different links.
+ */
+ lazyBuildSourceFiles() {
+ return new Promise((resolve, reject) => {
+ logger.info('Generating landing page...');
+ this.generateLandingPage()
+ .then(() => {
+ const lazyLoadingSpinnerHtmlFilePath = path.join(__dirname, LAZY_LOADING_SITE_FILE_NAME);
+ const outputSpinnerHtmlFilePath = path.join(this.outputPath, LAZY_LOADING_SITE_FILE_NAME);
+
+ return fs.copyAsync(lazyLoadingSpinnerHtmlFilePath, outputSpinnerHtmlFilePath);
+ })
+ .then(() => fs.removeAsync(this.tempPath))
+ .then(() => this.lazyBuildAllPagesNotViewed())
+ .then(() => logger.info('Landing page built, other pages will be built as you navigate to them!'))
+ .then(resolve)
+ .catch((error) => {
+ // if error, remove the site and temp folders
+ Site.rejectHandler(reject, error, [this.tempPath, this.outputPath]);
+ });
+ });
+ }
+
_rebuildAffectedSourceFiles(filePaths) {
const filePathArray = Array.isArray(filePaths) ? filePaths : [filePaths];
const uniquePaths = _.uniq(filePathArray);
@@ -581,14 +653,73 @@ class Site {
});
}
+ _rebuildPageBeingViewed(normalizedUrls) {
+ const startTime = new Date();
+ const normalizedUrlArray = Array.isArray(normalizedUrls) ? normalizedUrls : [normalizedUrls];
+ const uniqueUrls = _.uniq(normalizedUrlArray);
+ uniqueUrls.forEach(normalizedUrl => logger.info(
+ `Building ${normalizedUrl} as some of its dependencies were changed since the last visit`));
+ MarkBind.resetVariables();
+
+ /*
+ Lazy loading only builds the page being viewed, but the user may be quick enough
+ to trigger multiple page builds before the first one has finished building,
+ hence we need to take this into account.
+ */
+ const regeneratePagesBeingViewed = uniqueUrls.map(normalizedUrl =>
+ new Promise((resolve, reject) => {
+ this._setTimestampVariable();
+ const pageToRebuild = this.pages.find(page =>
+ FsUtil.removeExtension(page.sourcePath) === normalizedUrl);
+
+ if (!pageToRebuild) {
+ Site.rejectHandler(reject,
+ new Error(`Failed to rebuild ${normalizedUrl} during lazy loading`),
+ [this.tempPath, this.outputPath]);
+ }
+
+ this.toRebuild.delete(normalizedUrl);
+ pageToRebuild.userDefinedVariablesMap = this.userDefinedVariablesMap;
+ pageToRebuild.generate(new Set())
+ .then(() => {
+ pageToRebuild.collectHeadingsAndKeywords();
+
+ return this.writeSiteData();
+ })
+ .then(() => {
+ const endTime = new Date();
+ const totalBuildTime = (endTime - startTime) / 1000;
+ logger.info(`Lazy website regeneration complete! Total build time: ${totalBuildTime}s`);
+ })
+ .then(resolve)
+ .catch((error) => {
+ logger.error(error);
+ reject(new Error(`Failed to rebuild ${normalizedUrl} during lazy loading`));
+ });
+ }),
+ );
+
+ return Promise.all(regeneratePagesBeingViewed)
+ .then(() => fs.removeAsync(this.tempPath));
+ }
+
_rebuildSourceFiles() {
- logger.warn('Rebuilding all source files');
+ logger.info('File added or removed, updating list of site\'s pages...');
return new Promise((resolve, reject) => {
Promise.resolve('')
.then(() => this.updateAddressablePages())
- // ignore the warning on next line as IDE doesn't understand `delay` very well
.then(filesToRemove => this.removeAsset(filesToRemove))
- .then(() => this.buildSourceFiles())
+ .then(() => {
+ if (this.onePagePath) {
+ this.mapAddressablePagesToPages(this.addressablePages || [], this.getFavIconUrl());
+
+ return this.rebuildPageBeingViewed(this.currentPageViewed)
+ .then(() => this.lazyBuildAllPagesNotViewed());
+ }
+
+ logger.warn('Rebuilding all pages...');
+ return this.buildSourceFiles();
+ })
.then(resolve)
.catch((error) => {
// if error, remove the site and temp folders
@@ -740,6 +871,33 @@ class Site {
.forEach(plugin => this.loadPlugin(plugin, true));
}
+ getFavIconUrl() {
+ const { baseUrl, faviconPath } = this.siteConfig;
+
+ if (faviconPath) {
+ if (!fs.existsSync(path.join(this.rootPath, faviconPath))) {
+ logger.warn(`${faviconPath} does not exist`);
+ }
+ return url.join('/', baseUrl, faviconPath);
+ } else if (fs.existsSync(path.join(this.rootPath, FAVICON_DEFAULT_PATH))) {
+ return url.join('/', baseUrl, FAVICON_DEFAULT_PATH);
+ }
+
+ return undefined;
+ }
+
+ mapAddressablePagesToPages(addressablePages, faviconUrl) {
+ this.pages = addressablePages.map(page => this.createPage({
+ faviconUrl,
+ pageSrc: page.src,
+ title: page.title,
+ layout: page.layout,
+ frontmatter: page.frontmatter,
+ searchable: page.searchable !== 'no',
+ externalScripts: page.externalScripts,
+ }));
+ }
+
/**
* Collects the special tags of the site's plugins, and injects them into the parsers.
*/
@@ -775,47 +933,14 @@ class Site {
generatePages() {
// Run MarkBind include and render on each source file.
// Render the final rendered page to the output folder.
- const { baseUrl, faviconPath } = this.siteConfig;
const addressablePages = this.addressablePages || [];
const builtFiles = new Set();
const processingFiles = [];
- let faviconUrl;
- if (faviconPath) {
- faviconUrl = url.join('/', baseUrl, faviconPath);
- if (!fs.existsSync(path.join(this.rootPath, faviconPath))) {
- logger.warn(`${faviconPath} does not exist`);
- }
- } else if (fs.existsSync(path.join(this.rootPath, FAVICON_DEFAULT_PATH))) {
- faviconUrl = url.join('/', baseUrl, FAVICON_DEFAULT_PATH);
- }
+ const faviconUrl = this.getFavIconUrl();
this._setTimestampVariable();
- if (this.onePagePath) {
- const page = addressablePages.find(p => p.src === this.onePagePath);
- if (!page) {
- return Promise.reject(new Error(`${this.onePagePath} is not specified in the site configuration.`));
- }
- this.pages.push(this.createPage({
- faviconUrl,
- pageSrc: page.src,
- title: page.title,
- layout: page.layout,
- frontmatter: page.frontmatter,
- searchable: page.searchable !== 'no',
- externalScripts: page.externalScripts,
- }));
- } else {
- this.pages = addressablePages.map(page => this.createPage({
- faviconUrl,
- pageSrc: page.src,
- title: page.title,
- layout: page.layout,
- frontmatter: page.frontmatter,
- searchable: page.searchable !== 'no',
- externalScripts: page.externalScripts,
- }));
- }
+ this.mapAddressablePagesToPages(addressablePages, faviconUrl);
const progressBar = new ProgressBar(`[:bar] :current / ${this.pages.length} pages built`,
{ total: this.pages.length });
@@ -835,7 +960,27 @@ class Site {
});
}
+ /**
+ * Renders only the starting page for lazy loading to the output folder.
+ */
+ generateLandingPage() {
+ const addressablePages = this.addressablePages || [];
+ const faviconUrl = this.getFavIconUrl();
+
+ this._setTimestampVariable();
+ this.mapAddressablePagesToPages(addressablePages, faviconUrl);
+
+ const landingPage = this.pages.find(page => page.src === this.onePagePath);
+ if (!landingPage) {
+ return Promise.reject(new Error(`${this.onePagePath} is not specified in the site configuration.`));
+ }
+
+ return landingPage.generate(new Set());
+ }
+
regenerateAffectedPages(filePaths) {
+ const startTime = new Date();
+
const builtFiles = new Set();
const processingFiles = [];
const shouldRebuildAllPages = this.collectUserDefinedVariablesMapIfNeeded(filePaths) || this.forceReload;
@@ -844,13 +989,25 @@ class Site {
}
this._setTimestampVariable();
this.pages.forEach((page) => {
- if (shouldRebuildAllPages || filePaths.some((filePath) => {
+ const doFilePathsHaveSourceFiles = filePaths.some((filePath) => {
const isIncludedFile = page.includedFiles.has(filePath);
const isPluginSourceFile = page.pluginSourceFiles.has(filePath);
return isIncludedFile || isPluginSourceFile;
- })) {
- // eslint-disable-next-line no-param-reassign
+ });
+
+ if (shouldRebuildAllPages || doFilePathsHaveSourceFiles) {
+ if (this.onePagePath) {
+ const normalizedSource = FsUtil.removeExtension(page.sourcePath);
+ const isPageBeingViewed = normalizedSource === this.currentPageViewed;
+
+ if (!isPageBeingViewed) {
+ this.toRebuild.add(normalizedSource);
+ return;
+ }
+ }
+
+ // eslint-disable-next-line no-param-reassign
page.userDefinedVariablesMap = this.userDefinedVariablesMap;
processingFiles.push(page.generate(builtFiles)
.catch((err) => {
@@ -864,8 +1021,22 @@ class Site {
return new Promise((resolve, reject) => {
Promise.all(processingFiles)
- .then(() => this.updateSiteData(shouldRebuildAllPages ? undefined : filePaths))
+ .then(() => {
+ // For lazy loading, we defer updating site data pages not being viewed,
+ // even if all pages should be rebuilt, until they are navigated to.
+ const shouldUpdateAllSiteData = shouldRebuildAllPages && !this.onePagePath;
+ return this.updateSiteData(shouldUpdateAllSiteData ? undefined : filePaths);
+ })
.then(() => logger.info('Pages rebuilt'))
+ .then(() => {
+ const endTime = new Date();
+ const totalBuildTime = (endTime - startTime) / 1000;
+ logger.info(`Website regeneration complete! Total build time: ${totalBuildTime}s`);
+ if (!this.onePagePath && totalBuildTime > LAZY_LOADING_REBUILD_TIME_RECOMMENDATION_LIMIT) {
+ logger.info('Your pages took quite a while to rebuild...'
+ + 'Have you considered using markbind serve -o when writing content to speed things up?');
+ }
+ })
.then(resolve)
.catch(reject);
});
@@ -880,8 +1051,9 @@ class Site {
*/
updateSiteData(filePaths) {
const generateForAllPages = filePaths === undefined;
+ const filePathsToUpdateData = Array.isArray(filePaths) ? filePaths : [filePaths];
this.pages.forEach((page) => {
- if (generateForAllPages || filePaths.some(filePath => page.includedFiles.has(filePath))) {
+ if (generateForAllPages || filePathsToUpdateData.some(filePath => page.includedFiles.has(filePath))) {
page.collectHeadingsAndKeywords();
}
});
@@ -1071,6 +1243,9 @@ class Site {
*/
Site.prototype.buildAsset = delay(Site.prototype._buildMultipleAssets, 1000);
+Site.prototype.rebuildPageBeingViewed
+ = delay(Site.prototype._rebuildPageBeingViewed, 1000);
+
/**
* Rebuild pages that are affected by changes in filePaths
* @param filePaths a single path or an array of paths corresponding to the files that have changed
diff --git a/src/constants.js b/src/constants.js
index 35d7c0a41e..f89bf66f9f 100644
--- a/src/constants.js
+++ b/src/constants.js
@@ -56,6 +56,9 @@ module.exports = {
SITE_CONFIG_NAME: 'site.json',
SITE_DATA_NAME: 'siteData.json',
LAYOUT_SITE_FOLDER_NAME: 'layouts',
+ LAZY_LOADING_SITE_FILE_NAME: 'LazyLiveReloadLoadingSite.html',
+ LAZY_LOADING_BUILD_TIME_RECOMMENDATION_LIMIT: 30000,
+ LAZY_LOADING_REBUILD_TIME_RECOMMENDATION_LIMIT: 5000,
USER_VARIABLES_PATH: '_markbind/variables.md',
WIKI_SITE_NAV_PATH: '_Sidebar.md',
WIKI_FOOTER_PATH: '_Footer.md',