Skip to content

Commit

Permalink
feat: Express importing (adobe#8)
Browse files Browse the repository at this point in the history 
- introduce express transformation option
- automatically generate transformation file from current mapping
  • Loading branch information
@arumsey
arumsey authored May 24, 2024
1 parent 838ba88 commit 8cdeb20
Show file tree
Hide file tree
Showing 7 changed files with 233 additions and 78 deletions.
32 changes: 8 additions & 24 deletions import-manual-mapping.html → import-express.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@

<head>
<meta charset="utf-8" />
<title>Import - Manual Mapping Tool</title>
<title>Import - Express Transformations</title>
<link rel="stylesheet" href="./css/styles.css" />
<link rel="stylesheet" href="./css/import/import.css" />

Expand All @@ -31,10 +31,10 @@
<sp-theme color="dark" scale="medium">
<main>
<!-- IMPORT SECTION -->
<section class="import">
<section class="import import-express">
<div class="section-title">
<h2>Import - Manual Mapping</h2>
<p>Manually map page sections to existing block.</p>
<h2>Import - Express</h2>
<p>Express import a site using section mapping.</p>
<sp-divider size="l"></sp-divider>
</div>
<div class="section-cols">
Expand All @@ -48,22 +48,13 @@ <h2>Import - Manual Mapping</h2>
<sp-accordion>
<sp-accordion-item label="Load Options">
<div>
<sp-field-label for="import-file-url" required>Transformation file URL</sp-field-label>
<sp-textfield class="option-field locked" id="import-file-url" type="url" value="http://localhost:3001/tools/importer/helix-importer-ui/js/sections-mapping/sections-mapping.import.js" disabled></sp-textfield>

<sp-field-label for="import-pageload-timeout">Page load timeout</sp-field-label>
<sp-number-field class="option-field" id="import-pageload-timeout" value="100" min="0" step="100" format-options='{ "style": "unit", "unit": "millisecond", "unitDisplay": "short" }'></sp-number-field>

<div class="local-save">
<sp-checkbox class="option-field" id="import-local-docx" checked>
Save as docx
</sp-checkbox>
<sp-checkbox class="option-field" id="import-local-html">
Save as HTML
</sp-checkbox>
<sp-checkbox class="option-field" id="import-local-md">
Save as Markdown
</sp-checkbox>
</div>

<sp-checkbox class="option-field" id="import-enable-js">
Expand Down Expand Up @@ -99,8 +90,7 @@ <h3>Page preview</h3>
<sp-tabs selected="mapping-editor">
<sp-tab label="Mapping" value="mapping-editor"></sp-tab>
<sp-tab label="Preview" value="import-preview"></sp-tab>
<sp-tab label="Markdown" value="import-markdown"></sp-tab>
<sp-tab label="HTML" value="import-html"></sp-tab>
<sp-tab label="Transformation" value="import-transform"></sp-tab>
<div id="import-file-picker-container"></div>
<sp-tab-panel value="mapping-editor">
<sp-theme color="light" scale="medium">
Expand All @@ -122,23 +112,17 @@ <h2>&nbsp;</h2>
<div id="import-markdown-preview"></div>
</sp-theme>
</sp-tab-panel>
<sp-tab-panel value="import-markdown">
<div data-panel="source">
<div class="code">
<textarea id="import-markdown-source" rows="4" cols="10"></textarea>
</div>
</div>
</sp-tab-panel>
<sp-tab-panel value="import-html">
<sp-tab-panel value="import-transform">
<div data-panel="source">
<div class="code">
<textarea id="import-transformed-html" rows="4" cols="10"></textarea>
<textarea id="import-transform-source" rows="4" cols="10"></textarea>
</div>
</div>
</sp-tab-panel>
</sp-tabs>
<sp-button-group>
<sp-button id="import-downloadImportReport" class="hidden">Download import report</sp-button>
<sp-button id="import-downloadTransformation">Export transformation</sp-button>
</sp-button-group>
</div>
</div>
Expand Down
4 changes: 2 additions & 2 deletions index.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,9 +20,9 @@
<header><a href="#"><h1>AEM Importer</h1></a></header>
<nav>
<sp-sidenav defaultValue="import.html">
<sp-sidenav-item value="import-express.html" label="Import - Express"></sp-sidenav-item>
<sp-sidenav-item value="import.html" label="Import - Workbench" selected></sp-sidenav-item>
<sp-sidenav-item value="import-bulk.html" label="Import - Bulk"></sp-sidenav-item>
<sp-sidenav-item value="import-manual-mapping.html" label="Import - Manual Mapping"></sp-sidenav-item>
<sp-sidenav-item value="crawl.html" label="Crawl"></sp-sidenav-item>
<sp-sidenav-item value="inspect.html" label="Eyedropper"></sp-sidenav-item>
</sp-sidenav>
Expand All @@ -37,7 +37,7 @@
</aside>
<!-- MAIN CONTENT -->
<main>
<iframe src="import-manual-mapping.html"></iframe>
<iframe src="import-express.html"></iframe>
</main>
<div id="alert-container"></div>
</sp-theme>
Expand Down
160 changes: 114 additions & 46 deletions js/import/import.ui.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@ import PollImporter from '../shared/pollimporter.js';
import alert from '../shared/alert.js';
import { toggleLoadingButton } from '../shared/ui.js';
import { getImporterSectionsMapping, saveImporterSectionsMapping } from '../sections-mapping/utils.ui.js';
import { buildTransformationConfigFromMapping } from "./utils.import.js";
import { TransformFactory } from "../shared/transformfactory.js";

const PARENT_SELECTOR = '.import';
const CONFIG_PARENT_SELECTOR = `${PARENT_SELECTOR} form`;
Expand All @@ -32,6 +34,7 @@ const FOLDERNAME_SPAN = document.getElementById('folder-name');
const TRANSFORMED_HTML_TEXTAREA = document.getElementById('import-transformed-html');
const MD_SOURCE_TEXTAREA = document.getElementById('import-markdown-source');
const MD_PREVIEW_PANEL = document.getElementById('import-markdown-preview');
const TRANSFORMATION_TEXTAREA = document.getElementById('import-transform-source');

const SPTABS = document.querySelector(`${PARENT_SELECTOR} sp-tabs`);

Expand All @@ -44,7 +47,9 @@ const BULK_URLS_LIST = document.querySelector('#import-result ul');
const IMPORT_FILE_PICKER_CONTAINER = document.getElementById('import-file-picker-container');

// manual mapping elements
const IS_EXPRESS = document.querySelector('.import-express') !== null;
const DETECT_BUTTON = document.getElementById('detect-sections-button');
const SAVE_TRANSFORMATION_BUTTON = document.getElementById('import-downloadTransformation');
const MAPPING_EDITOR_SECTIONS = document.getElementById('mapping-editor-sections');

const REPORT_FILENAME = 'import-report.xlsx';
Expand All @@ -57,49 +62,75 @@ let isSaveLocal = false;
let dirHandle = null;

const setupUI = () => {
ui.transformedEditor = CodeMirror.fromTextArea(TRANSFORMED_HTML_TEXTAREA, {
lineNumbers: true,
mode: 'htmlmixed',
theme: 'base16-dark',
});
ui.transformedEditor.setSize('100%', '100%');
if (TRANSFORMED_HTML_TEXTAREA) {
ui.transformedEditor = CodeMirror.fromTextArea(TRANSFORMED_HTML_TEXTAREA, {
lineNumbers: true,
mode: 'htmlmixed',
theme: 'base16-dark',
});
ui.transformedEditor.setSize('100%', '100%');
}

ui.markdownEditor = CodeMirror.fromTextArea(MD_SOURCE_TEXTAREA, {
lineNumbers: true,
mode: 'markdown',
theme: 'base16-dark',
});
ui.markdownEditor.setSize('100%', '100%');
if (MD_SOURCE_TEXTAREA) {
ui.markdownEditor = CodeMirror.fromTextArea(MD_SOURCE_TEXTAREA, {
lineNumbers: true,
mode: 'markdown',
theme: 'base16-dark',
});
ui.markdownEditor.setSize('100%', '100%');
}

if (TRANSFORMATION_TEXTAREA) {
ui.jsonEditor = CodeMirror.fromTextArea(TRANSFORMATION_TEXTAREA, {
lineNumbers: false,
mode: 'javascript',
json: true,
readOnly: true,
theme: 'base16-dark',
});
ui.jsonEditor.setSize('100%', '100%');
}

ui.markdownPreview = MD_PREVIEW_PANEL;
// XSS review: we need interpreted HTML here - <script> tags are removed by importer anyway
ui.markdownPreview.innerHTML = WebImporter.md2html('Run an import to see some markdown.');
if (MD_PREVIEW_PANEL) {
ui.markdownPreview = MD_PREVIEW_PANEL;
// XSS review: we need interpreted HTML here - <script> tags are removed by importer anyway
ui.markdownPreview.innerHTML = WebImporter.md2html('Run an import to see some markdown.');
}

SPTABS.selected = 'mapping-editor';
};

const loadResult = ({ md, html: outputHTML }) => {
const loadResult = ({ md, html: outputHTML }, originalURL) => {
if (outputHTML) {
ui.transformedEditor.setValue(html_beautify(outputHTML.replaceAll(/\s+/g, ' '), {
ui.transformedEditor?.setValue(html_beautify(outputHTML.replaceAll(/\s+/g, ' '), {
indent_size: '2',
}));
}

if (md) {
ui.markdownEditor.setValue(md || '');

const mdPreview = WebImporter.md2html(md);
// XSS review: we need interpreted HTML here - <script> tags are removed by importer anyway
ui.markdownPreview.innerHTML = mdPreview;

// remove existing classes and styles
Array.from(ui.markdownPreview.querySelectorAll('[class], [style]')).forEach((t) => {
t.removeAttribute('class');
t.removeAttribute('style');
});
ui.markdownEditor?.setValue(md || '');

if (ui.markdownPreview) {
const mdPreview = WebImporter.md2html(md);
// XSS review: we need interpreted HTML here - <script> tags are removed by importer anyway
ui.markdownPreview.innerHTML = mdPreview;
// remove existing classes and styles
Array.from(ui.markdownPreview.querySelectorAll('[class], [style]')).forEach((t) => {
t.removeAttribute('class');
t.removeAttribute('style');
});
}
} else {
ui.markdownEditor.setValue('No preview available.');
ui.markdownPreview.innerHTML = 'No preview available.';
ui.markdownEditor?.setValue('No preview available.');
if (ui.markdownPreview) {
ui.markdownPreview.innerHTML = 'No preview available.';
}
}

if (ui.jsonEditor) {
const mapping = getImporterSectionsMapping(originalURL) || [];
const transform = buildTransformationConfigFromMapping(mapping);
ui.jsonEditor.setValue(JSON.stringify(transform, null, 2));
}
};

Expand Down Expand Up @@ -137,11 +168,11 @@ const updateImporterUI = (results, originalURL) => {
if (results.length > 0) {
picker.addEventListener('change', (e) => {
const r = results.filter((i) => i.path === e.target.value)[0];
loadResult(r);
loadResult(r, originalURL);
});
}

loadResult(results[0]);
loadResult(results[0], originalURL);
} else if (status === 'redirect') {
alert.warning(`No page imported: ${results[0].from} redirects to ${results[0].to}`);
}
Expand Down Expand Up @@ -201,11 +232,21 @@ const initImportStatus = () => {
};

const disableProcessButtons = () => {
IMPORT_BUTTON.disabled = true;
if (IMPORT_BUTTON) {
IMPORT_BUTTON.disabled = true;
}
if (DETECT_BUTTON) {
DETECT_BUTTON.disabled = true;
}
};

const enableProcessButtons = () => {
IMPORT_BUTTON.disabled = false;
if (IMPORT_BUTTON) {
IMPORT_BUTTON.disabled = false;
}
if (DETECT_BUTTON) {
DETECT_BUTTON.disabled = false;
}
};

const getProxyURLSetup = (url, origin) => {
Expand Down Expand Up @@ -635,7 +676,7 @@ const attachListeners = () => {
await postImportStep();
});

IMPORT_BUTTON.addEventListener('click', (async () => {
IMPORT_BUTTON?.addEventListener('click', (async () => {
initImportStatus();

if (IS_BULK) {
Expand All @@ -645,10 +686,10 @@ const attachListeners = () => {
} else {
PREVIEW_CONTAINER.classList.add('hidden');
}
DOWNLOAD_IMPORT_REPORT_BUTTON.classList.remove('hidden');
DOWNLOAD_IMPORT_REPORT_BUTTON?.classList.remove('hidden');
} else {
DOWNLOAD_IMPORT_REPORT_BUTTON.classList.add('hidden');
PREVIEW_CONTAINER.classList.remove('hidden');
DOWNLOAD_IMPORT_REPORT_BUTTON?.classList.add('hidden');
SAVE_TRANSFORMATION_BUTTON.classList.add('hidden');
}

disableProcessButtons();
Expand Down Expand Up @@ -746,11 +787,18 @@ const attachListeners = () => {
});

if (onLoadSucceeded) {
let transform = null;
if (IS_EXPRESS) {
// auto generate transformation config
const mapping = getImporterSectionsMapping(originalURL) || [];
transform = TransformFactory.create(buildTransformationConfigFromMapping(mapping));
}
config.importer.setTransformationInput({
url: replacedURL,
document: frame.contentDocument,
includeDocx,
params: { originalURL },
transform,
});
await config.importer.transform();
}
Expand Down Expand Up @@ -809,18 +857,18 @@ const attachListeners = () => {
} else {
const frame = getContentFrame();
frame.removeEventListener('transformation-complete', processNext);
DOWNLOAD_IMPORT_REPORT_BUTTON.classList.remove('hidden');
DOWNLOAD_IMPORT_REPORT_BUTTON?.classList.remove('hidden');
enableProcessButtons();
toggleLoadingButton(IMPORT_BUTTON);
}
};
processNext();
}));

DETECT_BUTTON.addEventListener('click', (async () => {
DETECT_BUTTON?.addEventListener('click', (async () => {
initImportStatus();
DOWNLOAD_IMPORT_REPORT_BUTTON.classList.add('hidden');
PREVIEW_CONTAINER.classList.remove('hidden');
DOWNLOAD_IMPORT_REPORT_BUTTON?.classList.add('hidden');
PREVIEW_CONTAINER?.classList.remove('hidden');
MAPPING_EDITOR_SECTIONS.innerHTML = '';

disableProcessButtons();
Expand Down Expand Up @@ -980,13 +1028,32 @@ const attachListeners = () => {
processNext();
}));

IMPORTFILEURL_FIELD.addEventListener('change', async (event) => {
SAVE_TRANSFORMATION_BUTTON?.addEventListener('click', async () => {
const originalURL = config.fields['import-url']

const importDirHandle = await getDirectoryHandle();
await importDirHandle.requestPermission({
mode: 'readwrite',
});

const mapping = getImporterSectionsMapping(originalURL) || [];

// save sections mapping data for current URL
await saveFile(importDirHandle, 'import_mapping.json', JSON.stringify(mapping, null, 2));

// save import json
const transformCfg = buildTransformationConfigFromMapping(mapping);
await saveFile(importDirHandle, 'import.json', JSON.stringify(transformCfg, null, 2));

});

IMPORTFILEURL_FIELD?.addEventListener('change', async (event) => {
if (config.importer) {
await config.importer.setImportFileURL(event.target.value);
}
});

DOWNLOAD_IMPORT_REPORT_BUTTON.addEventListener('click', (async () => {
DOWNLOAD_IMPORT_REPORT_BUTTON?.addEventListener('click', (async () => {
const buffer = await getReport();
const a = document.createElement('a');
const blob = new Blob([buffer], { type: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet' });
Expand All @@ -999,8 +1066,9 @@ const attachListeners = () => {
SPTABS.addEventListener('change', () => {
// required for code to load in editors
setTimeout(() => {
ui.transformedEditor.refresh();
ui.markdownEditor.refresh();
ui.transformedEditor?.refresh();
ui.markdownEditor?.refresh();
ui.jsonEditor?.refresh();
}, 1);
});
}
Expand Down
Loading

0 comments on commit 8cdeb20

Please sign in to comment.