From 899c3ee0c3479bb2495248cbb82a12a8c3e1ba20 Mon Sep 17 00:00:00 2001 From: genefy Date: Mon, 22 Nov 2021 17:50:53 +0800 Subject: [PATCH 1/3] feat: add build concurrency control --- packages/@vuepress/core/lib/node/build/index.js | 16 ++++++++++++---- packages/vuepress/lib/registerCoreCommands.js | 1 + 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/packages/@vuepress/core/lib/node/build/index.js b/packages/@vuepress/core/lib/node/build/index.js index 80afc9d7f9..37754a77c0 100644 --- a/packages/@vuepress/core/lib/node/build/index.js +++ b/packages/@vuepress/core/lib/node/build/index.js @@ -20,6 +20,7 @@ module.exports = class Build extends EventEmitter { constructor (context) { super() this.context = context + this.maxConcurrency = this.context.options.maxConcurrency this.outDir = this.context.outDir } @@ -90,10 +91,17 @@ module.exports = class Build extends EventEmitter { // render pages logger.wait('Rendering static HTML...') - - const pagePaths = await Promise.all( - this.context.pages.map(page => this.renderPage(page)) - ) + if (this.maxConcurrency) logger.info(`max concurrency set: ${this.maxConcurrency}`) + + const pagePaths = [] + while (this.context.pages.length) { + // 100 at a time + const segmentPaths = await Promise.all( + this.context.pages.splice(0, this.maxConcurrency || 100000) + .map(page => this.renderPage(page)) + ) + pagePaths.push(...segmentPaths) + } readline.clearLine(process.stdout, 0) readline.cursorTo(process.stdout, 0) diff --git a/packages/vuepress/lib/registerCoreCommands.js b/packages/vuepress/lib/registerCoreCommands.js index 2066565e97..3906c3f920 100644 --- a/packages/vuepress/lib/registerCoreCommands.js +++ b/packages/vuepress/lib/registerCoreCommands.js @@ -51,6 +51,7 @@ module.exports = function (cli, options) { .option('--no-cache', 'clean the cache before build') .option('--debug', 'build in development mode for debugging') .option('--silent', 'build static site in silent mode') + .option('--max-concurrency', 'set the max docs concurrently processed when build static site') .action((sourceDir = '.', commandOptions) => { const { debug, silent } = commandOptions From 6676550f72abfd88c82bbba6df5b57aabe9b95a2 Mon Sep 17 00:00:00 2001 From: genefy Date: Wed, 24 Nov 2021 20:01:17 +0800 Subject: [PATCH 2/3] fix: rm comment --- packages/@vuepress/core/lib/node/build/index.js | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/@vuepress/core/lib/node/build/index.js b/packages/@vuepress/core/lib/node/build/index.js index 37754a77c0..11cda26c3b 100644 --- a/packages/@vuepress/core/lib/node/build/index.js +++ b/packages/@vuepress/core/lib/node/build/index.js @@ -95,7 +95,6 @@ module.exports = class Build extends EventEmitter { const pagePaths = [] while (this.context.pages.length) { - // 100 at a time const segmentPaths = await Promise.all( this.context.pages.splice(0, this.maxConcurrency || 100000) .map(page => this.renderPage(page)) From bde8ac08cd5e3990b498021f59c517efcc73b3f0 Mon Sep 17 00:00:00 2001 From: genefy Date: Fri, 24 Dec 2021 15:49:53 +0800 Subject: [PATCH 3/3] feat: docs --- packages/@vuepress/core/lib/node/build/index.js | 9 ++++++--- packages/docs/docs/api/cli.md | 4 ++++ packages/docs/docs/zh/api/cli.md | 3 +++ 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/packages/@vuepress/core/lib/node/build/index.js b/packages/@vuepress/core/lib/node/build/index.js index 11cda26c3b..7f75850f5b 100644 --- a/packages/@vuepress/core/lib/node/build/index.js +++ b/packages/@vuepress/core/lib/node/build/index.js @@ -91,12 +91,15 @@ module.exports = class Build extends EventEmitter { // render pages logger.wait('Rendering static HTML...') - if (this.maxConcurrency) logger.info(`max concurrency set: ${this.maxConcurrency}`) + // If maxConcurrency is set, instead of rendering all pages concurrently, + // build task would render pages by smaller group to prevent OOM. + if (this.maxConcurrency) logger.info(`max concurrency set: ${this.maxConcurrency}`) const pagePaths = [] - while (this.context.pages.length) { + const maxConcurrency = this.maxConcurrency || 100000 + for (let i = 0; i < this.context.pages.length; i += maxConcurrency) { const segmentPaths = await Promise.all( - this.context.pages.splice(0, this.maxConcurrency || 100000) + this.context.pages.slice(i, i + maxConcurrency) .map(page => this.renderPage(page)) ) pagePaths.push(...segmentPaths) diff --git a/packages/docs/docs/api/cli.md b/packages/docs/docs/api/cli.md index 9f26190b6e..4fce287ff0 100644 --- a/packages/docs/docs/api/cli.md +++ b/packages/docs/docs/api/cli.md @@ -54,6 +54,10 @@ Start development server in debug mode. Start development server in silent mode. +#### --max-concurrency + +Set the max concurrency for rendering pages, to prevent OOM when rendering massive docs. + ### dev Start a development server. All options from `vuepress build` are available. And there are several options specifically for dev: diff --git a/packages/docs/docs/zh/api/cli.md b/packages/docs/docs/zh/api/cli.md index 9e44810b4f..25a457cf97 100644 --- a/packages/docs/docs/zh/api/cli.md +++ b/packages/docs/docs/zh/api/cli.md @@ -29,6 +29,9 @@ vuepress targetDir [options] ### --silent 以安静模式启动开发服务器。 +### --max-concurrency +设置渲染文档的最大并发量，当渲染大量文档，可能造成内存溢出时使用 + ## dev 启动一个开发服务器。来自 `vuepress build` 的所有选项都可用。除此以外，还有几个专门针对 dev 的选项：