From ecff249fa92c868aaf8ab3fec2c30a183f775574 Mon Sep 17 00:00:00 2001 From: Void <57257404+into-the-v0id@users.noreply.github.com> Date: Thu, 24 Oct 2024 19:37:06 +0200 Subject: [PATCH] Add 'gzip' plugin #680 --- CHANGELOG.md | 2 + core/utils/lume_config.ts | 1 + deps/streams.ts | 1 + plugins/gzip.ts | 54 + tests/__snapshots__/gzip.test.ts.snap | 1866 +++++++++++++++++++++++++ tests/gzip.test.ts | 33 + tests/plugins.test.ts | 1 + 7 files changed, 1958 insertions(+) create mode 100644 deps/streams.ts create mode 100644 plugins/gzip.ts create mode 100644 tests/__snapshots__/gzip.test.ts.snap create mode 100644 tests/gzip.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 93193e1c..2f8dd984 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ Go to the `v1` branch to see the changelog of Lume 1. - New plugin: `check_urls` to detect broken links [#675]. - New plugin: `icons` to load automatically icons from popular icon catalogs. - New plugin: `google_fonts` to download and self-host automatically fonts from Google Fonts. +- New plugin: `gzip` to compress files. [#680] ### Fixed - Nav plugin: Breadcrumb with urls with CJK characters. @@ -564,6 +565,7 @@ Go to the `v1` branch to see the changelog of Lume 1. [#677]: https://github.com/lumeland/lume/issues/677 [#678]: https://github.com/lumeland/lume/issues/678 [#679]: https://github.com/lumeland/lume/issues/679 +[#680]: https://github.com/lumeland/lume/issues/680 [Unreleased]: https://github.com/lumeland/lume/compare/v2.3.3...HEAD [2.3.3]: https://github.com/lumeland/lume/compare/v2.3.2...v2.3.3 diff --git a/core/utils/lume_config.ts b/core/utils/lume_config.ts index cf0a3c50..1242508c 100644 --- a/core/utils/lume_config.ts +++ b/core/utils/lume_config.ts @@ -13,6 +13,7 @@ export const pluginNames = [ "fff", "filter_pages", "google_fonts", + "gzip", "inline", "jsx", "jsx_preact", diff --git a/deps/streams.ts b/deps/streams.ts new file mode 100644 index 00000000..19b8a0b1 --- /dev/null +++ b/deps/streams.ts @@ -0,0 +1 @@ +export * from "jsr:@std/streams@1.0.7"; diff --git a/plugins/gzip.ts b/plugins/gzip.ts new file mode 100644 index 00000000..5b255a6a --- /dev/null +++ b/plugins/gzip.ts @@ -0,0 +1,54 @@ +import { merge } from "../core/utils/object.ts"; +import { concurrent } from "../core/utils/concurrent.ts"; +import { Page } from "../core/file.ts"; +import { toArrayBuffer } from "../deps/streams.ts"; + +import type { Extensions } from "../core/utils/path.ts"; +import type { Content } from "../core/file.ts"; +import type Site from "../core/site.ts"; + +export interface Options { + /** The list of extensions this plugin applies to */ + extensions?: Extensions; +} + +// Default options +export const defaults: Options = { + extensions: [".html", ".css", ".js", ".mjs", ".svg", ".json", ".xml", ".txt"], +}; + +/** + * A plugin to compress files with gzip + */ +export function gzip(userOptions?: Options) { + const options = merge(defaults, userOptions); + + return (site: Site) => { + const textEncoder = new TextEncoder(); + + site.process( + options.extensions, + (pages, allPages) => + concurrent(pages, async (page: Page) => { + const content = page.content as Content; + const contentStream = ReadableStream.from([ + typeof content === "string" ? textEncoder.encode(content) : content, + ]); + + const compressedStream = contentStream.pipeThrough( + new CompressionStream("gzip"), + ); + const compressedArrayBuffer = await toArrayBuffer(compressedStream); + const compressedContent = new Uint8Array(compressedArrayBuffer); + + const compressedPage = Page.create({ + url: page.outputPath + ".gz", + content: compressedContent, + }); + allPages.push(compressedPage); + }), + ); + }; +} + +export default gzip; diff --git a/tests/__snapshots__/gzip.test.ts.snap b/tests/__snapshots__/gzip.test.ts.snap new file mode 100644 index 00000000..b66c47b5 --- /dev/null +++ b/tests/__snapshots__/gzip.test.ts.snap @@ -0,0 +1,1866 @@ +export const snapshot = {}; + +snapshot[`gzip plugin 1`] = ` +{ + formats: [ + { + engines: 0, + ext: ".page.toml", + loader: [AsyncFunction: toml], + pageType: "page", + }, + { + engines: 1, + ext: ".page.ts", + loader: [AsyncFunction: module], + pageType: "page", + }, + { + engines: 1, + ext: ".page.js", + loader: [AsyncFunction: module], + pageType: "page", + }, + { + engines: 0, + ext: ".page.jsonc", + loader: [AsyncFunction: json], + pageType: "page", + }, + { + engines: 0, + ext: ".page.json", + loader: [AsyncFunction: json], + pageType: "page", + }, + { + assetLoader: [AsyncFunction: text], + dataLoader: [AsyncFunction: json], + engines: 0, + ext: ".json", + loader: [AsyncFunction: json], + pageType: "asset", + }, + { + dataLoader: [AsyncFunction: json], + engines: 0, + ext: ".jsonc", + loader: [AsyncFunction: json], + }, + { + engines: 1, + ext: ".md", + loader: [AsyncFunction: text], + pageType: "page", + }, + { + engines: 1, + ext: ".markdown", + loader: [AsyncFunction: text], + pageType: "page", + }, + { + dataLoader: [AsyncFunction: module], + engines: 1, + ext: ".js", + loader: [AsyncFunction: module], + }, + { + dataLoader: [AsyncFunction: module], + engines: 1, + ext: ".ts", + loader: [AsyncFunction: module], + }, + { + engines: 1, + ext: ".vento", + loader: [AsyncFunction: text], + pageType: "page", + }, + { + engines: 1, + ext: ".vto", + loader: [AsyncFunction: text], + pageType: "page", + }, + { + dataLoader: [AsyncFunction: toml], + engines: 0, + ext: ".toml", + loader: [AsyncFunction: toml], + }, + { + dataLoader: [AsyncFunction: yaml], + engines: 0, + ext: ".yaml", + loader: [AsyncFunction: yaml], + pageType: "page", + }, + { + dataLoader: [AsyncFunction: yaml], + engines: 0, + ext: ".yml", + loader: [AsyncFunction: yaml], + pageType: "page", + }, + { + assetLoader: [AsyncFunction: text], + engines: undefined, + ext: ".css", + pageType: "asset", + }, + { + copy: true, + engines: undefined, + ext: ".png", + }, + ], + src: [ + "/", + "/404.md", + "/_data.yml", + "/favicon.png", + "/images", + "/images/avatar.jpg", + "/page5.yaml", + "/pages", + "/pages/2020-06-21_page2.page.json", + "/pages/2021-01-02-18-32_page4.page.ts", + "/pages/_data", + "/pages/_data.yml", + "/pages/_data/colors.yml", + "/pages/_data/documents.ts", + "/pages/_data/drinks.js", + "/pages/_data/names.json", + "/pages/ghost", + "/pages/ghost/2021-12-29-page6.md", + "/pages/ghost/_data.yml", + "/pages/page1.md", + "/pages/page3.page.js", + "/pages/subpage", + "/pages/subpage/_data.yml", + "/pages/subpage/page7.page.js", + "/static.yml", + "/styles.css", + ], +} +`; + +snapshot[`gzip plugin 2`] = ` +[ + { + entry: "/favicon.png", + flags: [], + outputPath: "/favicon.png", + }, +] +`; + +snapshot[`gzip plugin 3`] = ` +[ + { + content: "Uint8Array(107)", + data: { + basename: "404", + content: "Uint8Array(107)", + page: [ + "src", + "data", + "asset", + ], + url: "/404.html.gz", + }, + src: { + asset: true, + ext: "", + path: "", + remote: undefined, + }, + }, + { + content: "Uint8Array(59)", + data: { + basename: "index", + content: "Uint8Array(59)", + page: [ + "src", + "data", + "asset", + ], + url: "/overrided-page2/index.html.gz", + }, + src: { + asset: true, + ext: "", + path: "", + remote: undefined, + }, + }, + { + content: "Uint8Array(59)", + data: { + basename: "index", + content: "Uint8Array(59)", + page: [ + "src", + "data", + "asset", + ], + url: "/page5/index.html.gz", + }, + src: { + asset: true, + ext: "", + path: "", + remote: undefined, + }, + }, + { + content: "Uint8Array(59)", + data: { + basename: "index", + content: "Uint8Array(59)", + page: [ + "src", + "data", + "asset", + ], + url: "/pages/new-name/page7/index.html.gz", + }, + src: { + asset: true, + ext: "", + path: "", + remote: undefined, + }, + }, + { + content: "Uint8Array(128)", + data: { + basename: "index", + content: "Uint8Array(128)", + page: [ + "src", + "data", + "asset", + ], + url: "/pages/page4/index.html.gz", + }, + src: { + asset: true, + ext: "", + path: "", + remote: undefined, + }, + }, + { + content: "Uint8Array(67)", + data: { + basename: "index", + content: "Uint8Array(67)", + page: [ + "src", + "data", + "asset", + ], + url: "/pages/page6/index.html.gz", + }, + src: { + asset: true, + ext: "", + path: "", + remote: undefined, + }, + }, + { + content: "Uint8Array(98)", + data: { + basename: "index", + content: "Uint8Array(98)", + page: [ + "src", + "data", + "asset", + ], + url: "/static/index.html.gz", + }, + src: { + asset: true, + ext: "", + path: "", + remote: undefined, + }, + }, + { + content: "Uint8Array(78)", + data: { + basename: "styles", + content: "Uint8Array(78)", + page: [ + "src", + "data", + "asset", + ], + url: "/styles.css.gz", + }, + src: { + asset: true, + ext: "", + path: "", + remote: undefined, + }, + }, + { + content: " +
This page is exported to /404.html
, not /404/index.html
This page is exported to /404.html
, not /404/index.html
Content of Page 6
+", + data: { + basename: "page6", + children: "Content of Page 6
+", + colors: "Array(3)", + content: "Content of Page 6 +", + date: [], + documents: "Array(3)", + drinks: [ + "alcoholic", + "others", + ], + imagick: "Array(1)", + mergedKeys: [ + "tags", + "metas", + "imagick", + ], + metas: [ + "title", + "description", + ], + names: "Array(2)", + page: [ + "src", + "data", + "asset", + ], + paginate: "paginate", + search: [], + site: "Folder overrided site name", + tags: "Array(2)", + title: "Page 6", + url: "/pages/page6/", + }, + src: { + asset: false, + ext: ".md", + path: "/pages/ghost/2021-12-29-page6", + remote: undefined, + }, + }, + { + content: "Content of Page 3", + data: { + basename: "page3", + colors: "Array(3)", + content: "Content of Page 3", + date: [], + documents: "Array(3)", + drinks: [ + "alcoholic", + "others", + ], + imagick: "Array(1)", + mergedKeys: [ + "tags", + "metas", + "imagick", + ], + metas: [ + "title", + "description", + ], + names: "Array(2)", + page: [ + "src", + "data", + "asset", + ], + paginate: "paginate", + search: [], + site: "Folder overrided site name", + tags: "Array(1)", + title: "Page 3", + url: "/page_3", + }, + src: { + asset: false, + ext: ".page.js", + path: "/pages/page3", + remote: undefined, + }, + }, + { + content: " +Content of Page 7", + data: { + basename: "page7", + children: "Content of Page 7", + colors: "Array(3)", + content: "Content of Page 7", + date: [], + documents: "Array(3)", + drinks: [ + "alcoholic", + "others", + ], + imagick: "Array(1)", + mergedKeys: [ + "tags", + "metas", + "imagick", + ], + metas: [ + "title", + "description", + ], + names: "Array(2)", + page: [ + "src", + "data", + "asset", + ], + paginate: "paginate", + search: [], + site: "Folder overrided site name", + tags: "Array(3)", + url: "/pages/new-name/page7/", + }, + src: { + asset: false, + ext: ".page.js", + path: "/pages/subpage/page7", + remote: undefined, + }, + }, + { + content: " +This yaml should be ignored because it's copied statically", + data: { + basename: "static", + children: "This yaml should be ignored because it's copied statically", + content: "This yaml should be ignored because it's copied statically", + date: [], + imagick: "Array(1)", + mergedKeys: [ + "tags", + "metas", + "imagick", + ], + metas: [ + "title", + "description", + ], + page: [ + "src", + "data", + "asset", + ], + paginate: "paginate", + search: [], + site: "Default site name", + tags: "Array(0)", + url: "/static/", + }, + src: { + asset: false, + ext: ".yml", + path: "/static", + remote: undefined, + }, + }, + { + content: "body { + font-family: sans-serif; + color: black; +} +", + data: { + basename: "styles", + content: "body { + font-family: sans-serif; + color: black; +} +", + date: [], + imagick: "Array(1)", + mergedKeys: [ + "tags", + "metas", + "imagick", + ], + metas: [ + "title", + "description", + ], + page: [ + "src", + "data", + "asset", + ], + paginate: "paginate", + search: [], + site: "Default site name", + tags: "Array(0)", + url: "/styles.css", + }, + src: { + asset: true, + ext: ".css", + path: "/styles", + remote: undefined, + }, + }, +] +`; + +snapshot[`gzip plugin with options 1`] = ` +{ + formats: [ + { + engines: 0, + ext: ".page.toml", + loader: [AsyncFunction: toml], + pageType: "page", + }, + { + engines: 1, + ext: ".page.ts", + loader: [AsyncFunction: module], + pageType: "page", + }, + { + engines: 1, + ext: ".page.js", + loader: [AsyncFunction: module], + pageType: "page", + }, + { + engines: 0, + ext: ".page.jsonc", + loader: [AsyncFunction: json], + pageType: "page", + }, + { + engines: 0, + ext: ".page.json", + loader: [AsyncFunction: json], + pageType: "page", + }, + { + dataLoader: [AsyncFunction: json], + engines: 0, + ext: ".json", + loader: [AsyncFunction: json], + }, + { + dataLoader: [AsyncFunction: json], + engines: 0, + ext: ".jsonc", + loader: [AsyncFunction: json], + }, + { + engines: 1, + ext: ".md", + loader: [AsyncFunction: text], + pageType: "page", + }, + { + engines: 1, + ext: ".markdown", + loader: [AsyncFunction: text], + pageType: "page", + }, + { + dataLoader: [AsyncFunction: module], + engines: 1, + ext: ".js", + loader: [AsyncFunction: module], + }, + { + dataLoader: [AsyncFunction: module], + engines: 1, + ext: ".ts", + loader: [AsyncFunction: module], + }, + { + engines: 1, + ext: ".vento", + loader: [AsyncFunction: text], + pageType: "page", + }, + { + engines: 1, + ext: ".vto", + loader: [AsyncFunction: text], + pageType: "page", + }, + { + dataLoader: [AsyncFunction: toml], + engines: 0, + ext: ".toml", + loader: [AsyncFunction: toml], + }, + { + dataLoader: [AsyncFunction: yaml], + engines: 0, + ext: ".yaml", + loader: [AsyncFunction: yaml], + pageType: "page", + }, + { + dataLoader: [AsyncFunction: yaml], + engines: 0, + ext: ".yml", + loader: [AsyncFunction: yaml], + pageType: "page", + }, + { + assetLoader: [AsyncFunction: text], + engines: undefined, + ext: ".css", + pageType: "asset", + }, + ], + src: [ + "/", + "/index.vto", + "/styles.css", + ], +} +`; + +snapshot[`gzip plugin with options 2`] = `[]`; + +snapshot[`gzip plugin with options 3`] = ` +[ + { + content: "Uint8Array(2390)", + data: { + basename: "styles", + content: "Uint8Array(2390)", + page: [ + "src", + "data", + "asset", + ], + url: "/styles.css.gz", + }, + src: { + asset: true, + ext: "", + path: "", + remote: undefined, + }, + }, + { + content: \` + + + + +A unique layout with a striking design
+Let's face it. Many cosmetics are bad for your skin. We use only natural ingredients and still provide consistently great tanning results.
+ Learn more +Lorem ipsum dolor sit amet, consectetur.
+Lorem ipsum dolor sit amet, consectetur.
+Lorem ipsum dolor sit amet, consectetur.
+Lorem ipsum dolor sit amet, consectetur.
+Lorem ipsum dolor sit amet, consectetur.
+Whether in the sun or on the couch, hydration is key to maintaining happy, healthy skin.
+ Shop now +Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.
+Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.
+Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.
+We're humans too and we understand that skin care and cosmetics should rejuvenate and rehydrate in the short and long run.
+ Shop now +A unique layout with a striking design
+Let's face it. Many cosmetics are bad for your skin. We use only natural ingredients and still provide consistently great tanning results.
+ Learn more +Lorem ipsum dolor sit amet, consectetur.
+Lorem ipsum dolor sit amet, consectetur.
+Lorem ipsum dolor sit amet, consectetur.
+Lorem ipsum dolor sit amet, consectetur.
+Lorem ipsum dolor sit amet, consectetur.
+Whether in the sun or on the couch, hydration is key to maintaining happy, healthy skin.
+ Shop now +Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.
+Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.
+Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.
+We're humans too and we understand that skin care and cosmetics should rejuvenate and rehydrate in the short and long run.
+ Shop now +A unique layout with a striking design
+Let's face it. Many cosmetics are bad for your skin. We use only natural ingredients and still provide consistently great tanning results.
+ Learn more +Lorem ipsum dolor sit amet, consectetur.
+Lorem ipsum dolor sit amet, consectetur.
+Lorem ipsum dolor sit amet, consectetur.
+Lorem ipsum dolor sit amet, consectetur.
+Lorem ipsum dolor sit amet, consectetur.
+Whether in the sun or on the couch, hydration is key to maintaining happy, healthy skin.
+ Shop now +Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.
+Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.
+Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.
+We're humans too and we understand that skin care and cosmetics should rejuvenate and rehydrate in the short and long run.
+ Shop now +