From a705befe8498fa993b34d6a64610e488c5e09dd8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0t=C4=9Bp=C3=A1n=20Gran=C3=A1t?= Date: Tue, 2 Jul 2024 14:32:05 +0200 Subject: [PATCH 1/3] fix: properly extract escaped characters and respect default namespace in magic comments --- src/extractor/parser/generalMapper.ts | 2 + src/extractor/parser/generateReport.ts | 2 +- .../parser/tokenMergers/stringMerger.ts | 45 ++++++----- src/test/unit/extractor/react/react.test.ts | 6 +- .../unit/extractor/shared/properties.test.ts | 78 +++++++++++++++++++ src/test/unit/extractor/shared/report.test.ts | 16 ++++ 6 files changed, 128 insertions(+), 21 deletions(-) diff --git a/src/extractor/parser/generalMapper.ts b/src/extractor/parser/generalMapper.ts index 1313fde..34164a6 100644 --- a/src/extractor/parser/generalMapper.ts +++ b/src/extractor/parser/generalMapper.ts @@ -23,6 +23,8 @@ export const generalMapper = (token: Token) => { case 'string.quoted.single.ts': case 'string.quoted.double.ts': return 'string.body'; + case 'constant.character.escape.ts': + return 'escaped.character'; // template strings case 'punctuation.definition.string.template.begin.ts': diff --git a/src/extractor/parser/generateReport.ts b/src/extractor/parser/generateReport.ts index 2b37267..41ee86b 100644 --- a/src/extractor/parser/generateReport.ts +++ b/src/extractor/parser/generateReport.ts @@ -37,7 +37,7 @@ function keyInfoFromComment( context.unusedComments.delete(commentAtLine); return { keyName: commentAtLine.keyName, - namespace: commentAtLine.namespace, + namespace: commentAtLine.namespace ?? context.options.defaultNamespace, defaultValue: commentAtLine.defaultValue, line: commentAtLine.line, }; diff --git a/src/extractor/parser/tokenMergers/stringMerger.ts b/src/extractor/parser/tokenMergers/stringMerger.ts index 91ed7ca..9ae4ddd 100644 --- a/src/extractor/parser/tokenMergers/stringMerger.ts +++ b/src/extractor/parser/tokenMergers/stringMerger.ts @@ -4,9 +4,7 @@ import { MachineType } from '../mergerMachine.js'; export const enum S { Idle, RegularString, - RegularStringEnd, TemplateString, - TemplateStringEnd, } export const stringMerger = { @@ -23,35 +21,46 @@ export const stringMerger = { break; case S.RegularString: if (type === 'string.body') { - return S.RegularStringEnd; + return S.RegularString; + } else if (type === 'escaped.character') { + return S.RegularString; } else if (type === 'string.end') { return end.MERGE_ALL; } break; - case S.RegularStringEnd: - if (type === 'string.end') { - return end.MERGE_ALL; - } - break; case S.TemplateString: if (type === 'string.template.body') { - return S.TemplateStringEnd; + return S.TemplateString; + } else if (type === 'escaped.character') { + return S.TemplateString; } else if (type === 'string.template.end') { return end.MERGE_ALL; } break; - case S.TemplateStringEnd: - if (type === 'string.template.end') { - return end.MERGE_ALL; - } } }, customType: 'string', resultToken: (matched) => { - if (matched.length === 3) { - return matched[1].token; - } else { - return ''; - } + return matched + .map((t) => { + switch (t.customType) { + case 'string.template.body': + case 'string.body': + return t.token; + case 'escaped.character': + if (t.token === "\\'") { + return "'"; + } + // interpret escape character + try { + return JSON.parse(`"${t.token}"`); + } catch (e) { + return t.token; + } + default: + return ''; + } + }) + .join(''); }, } as const satisfies MachineType; diff --git a/src/test/unit/extractor/react/react.test.ts b/src/test/unit/extractor/react/react.test.ts index 06531e2..51c8428 100644 --- a/src/test/unit/extractor/react/react.test.ts +++ b/src/test/unit/extractor/react/react.test.ts @@ -1377,7 +1377,7 @@ describe('magic comments', () => { it('overrides data from code (JSX-specific)', async () => { const expected = [ - { keyName: 'key-override-1', defaultValue: undefined, line: 6 }, + { keyName: 'key-override-1', namespace: 'namespace', line: 6 }, ]; const code = ` @@ -1392,7 +1392,9 @@ describe('magic comments', () => { } `; - const extracted = await extractReactKeys(code, 'test.jsx'); + const extracted = await extractReactKeys(code, 'test.jsx', { + defaultNamespace: 'namespace', + }); expect(extracted.keys).toEqual(expected); expect(extracted.warnings).toEqual([]); }); diff --git a/src/test/unit/extractor/shared/properties.test.ts b/src/test/unit/extractor/shared/properties.test.ts index b118224..9074496 100644 --- a/src/test/unit/extractor/shared/properties.test.ts +++ b/src/test/unit/extractor/shared/properties.test.ts @@ -138,6 +138,84 @@ describe('Plain JavaScript', () => { expect(extractValue(dict.value.key)).toBe('key'); expect(extractValue(dict.value.ns)).toBe(''); }); + + it('handles strings with newlines', async () => { + const dict = await getObject( + "const a = { key: 'key', defaultValue: 'default\\nvalue' }", + FILE_NAME, + 'react' + ); + + JSON.stringify(dict.value.defaultValue); + + expect(extractValue(dict.value.key)).toBe('key'); + expect(extractValue(dict.value.defaultValue)).toBe('default\nvalue'); + }); + + it('handles strings with newlines in template strings', async () => { + const dict = await getObject( + "const a = { key: 'key', defaultValue: `default\\nvalue` }", + FILE_NAME, + 'react' + ); + + JSON.stringify(dict.value.defaultValue); + + expect(extractValue(dict.value.key)).toBe('key'); + expect(extractValue(dict.value.defaultValue)).toBe('default\nvalue'); + }); + + it('handles strings with escaped backslash', async () => { + const dict = await getObject( + "const a = { key: 'key', defaultValue: `default\\\\value` }", + FILE_NAME, + 'react' + ); + + JSON.stringify(dict.value.defaultValue); + + expect(extractValue(dict.value.key)).toBe('key'); + expect(extractValue(dict.value.defaultValue)).toBe('default\\value'); + }); + + it('handles strings with escaped tab', async () => { + const dict = await getObject( + "const a = { key: 'key', defaultValue: `default\\tvalue` }", + FILE_NAME, + 'react' + ); + + JSON.stringify(dict.value.defaultValue); + + expect(extractValue(dict.value.key)).toBe('key'); + expect(extractValue(dict.value.defaultValue)).toBe('default\tvalue'); + }); + + it('handles strings with escaped quotes', async () => { + const dict = await getObject( + "const a = { key: 'key', defaultValue: `default\\'value` }", + FILE_NAME, + 'react' + ); + + JSON.stringify(dict.value.defaultValue); + + expect(extractValue(dict.value.key)).toBe('key'); + expect(extractValue(dict.value.defaultValue)).toBe("default'value"); + }); + + it('handles strings with escaped double quotes', async () => { + const dict = await getObject( + "const a = { key: 'key', defaultValue: `default\\\"value` }", + FILE_NAME, + 'react' + ); + + JSON.stringify(dict.value.defaultValue); + + expect(extractValue(dict.value.key)).toBe('key'); + expect(extractValue(dict.value.defaultValue)).toBe('default"value'); + }); }); }); }); diff --git a/src/test/unit/extractor/shared/report.test.ts b/src/test/unit/extractor/shared/report.test.ts index 52373d2..1f608bb 100644 --- a/src/test/unit/extractor/shared/report.test.ts +++ b/src/test/unit/extractor/shared/report.test.ts @@ -64,5 +64,21 @@ describe('report not influenced by surrounding block', () => { expect(report.keys).toEqual([{ keyName: 'keyName', line: 3 }]); }); + + it('respects default namespace in magic comments', async () => { + const report = await getReport( + ` + // @tolgee-key key-override + t('keyName') + `, + FILE_NAME, + 'react', + { defaultNamespace: 'namespace' } + ); + + expect(report.keys).toEqual([ + { keyName: 'key-override', namespace: 'namespace', line: 2 }, + ]); + }); }); }); From 99e538c70e50d23f86f8c2a375cc0def2e8ec7ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0t=C4=9Bp=C3=A1n=20Gran=C3=A1t?= Date: Tue, 2 Jul 2024 15:01:23 +0200 Subject: [PATCH 2/3] fix: properly extract escaped characters and respect default namespace in magic comments --- package-lock.json | 27 +++++++++++++++++++ package.json | 1 + .../parser/tokenMergers/stringMerger.ts | 16 ++++------- .../unit/extractor/shared/properties.test.ts | 19 +++++++++++-- 4 files changed, 50 insertions(+), 13 deletions(-) diff --git a/package-lock.json b/package-lock.json index a406965..00babad 100644 --- a/package-lock.json +++ b/package-lock.json @@ -18,6 +18,7 @@ "json5": "^2.2.3", "jsonschema": "^1.4.1", "openapi-fetch": "^0.9.7", + "unescape-js": "^1.1.4", "vscode-oniguruma": "^1.7.0", "vscode-textmate": "^9.0.0", "yauzl": "^2.10.0" @@ -10436,6 +10437,11 @@ "url": "https://github.com/chalk/strip-ansi?sponsor=1" } }, + "node_modules/string.fromcodepoint": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/string.fromcodepoint/-/string.fromcodepoint-0.2.1.tgz", + "integrity": "sha512-n69H31OnxSGSZyZbgBlvYIXlrMhJQ0dQAX1js1QDhpaUH6zmU3QYlj07bCwCNlPOu3oRXIubGPl2gDGnHsiCqg==" + }, "node_modules/strip-ansi": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", @@ -11150,6 +11156,14 @@ "node": ">=14.0" } }, + "node_modules/unescape-js": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/unescape-js/-/unescape-js-1.1.4.tgz", + "integrity": "sha512-42SD8NOQEhdYntEiUQdYq/1V/YHwr1HLwlHuTJB5InVVdOSbgI6xu8jK5q65yIzuFCfczzyDF/7hbGzVbyCw0g==", + "dependencies": { + "string.fromcodepoint": "^0.2.1" + } + }, "node_modules/unique-string": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/unique-string/-/unique-string-3.0.0.tgz", @@ -18882,6 +18896,11 @@ } } }, + "string.fromcodepoint": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/string.fromcodepoint/-/string.fromcodepoint-0.2.1.tgz", + "integrity": "sha512-n69H31OnxSGSZyZbgBlvYIXlrMhJQ0dQAX1js1QDhpaUH6zmU3QYlj07bCwCNlPOu3oRXIubGPl2gDGnHsiCqg==" + }, "strip-ansi": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", @@ -19390,6 +19409,14 @@ "@fastify/busboy": "^2.0.0" } }, + "unescape-js": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/unescape-js/-/unescape-js-1.1.4.tgz", + "integrity": "sha512-42SD8NOQEhdYntEiUQdYq/1V/YHwr1HLwlHuTJB5InVVdOSbgI6xu8jK5q65yIzuFCfczzyDF/7hbGzVbyCw0g==", + "requires": { + "string.fromcodepoint": "^0.2.1" + } + }, "unique-string": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/unique-string/-/unique-string-3.0.0.tgz", diff --git a/package.json b/package.json index fc8342a..03af859 100644 --- a/package.json +++ b/package.json @@ -37,6 +37,7 @@ "json5": "^2.2.3", "jsonschema": "^1.4.1", "openapi-fetch": "^0.9.7", + "unescape-js": "^1.1.4", "vscode-oniguruma": "^1.7.0", "vscode-textmate": "^9.0.0", "yauzl": "^2.10.0" diff --git a/src/extractor/parser/tokenMergers/stringMerger.ts b/src/extractor/parser/tokenMergers/stringMerger.ts index 9ae4ddd..3cf171d 100644 --- a/src/extractor/parser/tokenMergers/stringMerger.ts +++ b/src/extractor/parser/tokenMergers/stringMerger.ts @@ -1,5 +1,6 @@ import { GeneralTokenType } from '../generalMapper.js'; import { MachineType } from '../mergerMachine.js'; +import unescape from 'unescape-js'; export const enum S { Idle, @@ -41,26 +42,19 @@ export const stringMerger = { }, customType: 'string', resultToken: (matched) => { - return matched + const escaped = matched .map((t) => { switch (t.customType) { case 'string.template.body': case 'string.body': - return t.token; case 'escaped.character': - if (t.token === "\\'") { - return "'"; - } - // interpret escape character - try { - return JSON.parse(`"${t.token}"`); - } catch (e) { - return t.token; - } + return t.token; + default: return ''; } }) .join(''); + return unescape(escaped); }, } as const satisfies MachineType; diff --git a/src/test/unit/extractor/shared/properties.test.ts b/src/test/unit/extractor/shared/properties.test.ts index 9074496..e38c91c 100644 --- a/src/test/unit/extractor/shared/properties.test.ts +++ b/src/test/unit/extractor/shared/properties.test.ts @@ -193,7 +193,7 @@ describe('Plain JavaScript', () => { it('handles strings with escaped quotes', async () => { const dict = await getObject( - "const a = { key: 'key', defaultValue: `default\\'value` }", + "const a = { key: 'key', defaultValue: 'default\\'value' }", FILE_NAME, 'react' ); @@ -206,7 +206,7 @@ describe('Plain JavaScript', () => { it('handles strings with escaped double quotes', async () => { const dict = await getObject( - "const a = { key: 'key', defaultValue: `default\\\"value` }", + `const a = { key: "key", defaultValue: "default\\"value" }`, FILE_NAME, 'react' ); @@ -216,6 +216,21 @@ describe('Plain JavaScript', () => { expect(extractValue(dict.value.key)).toBe('key'); expect(extractValue(dict.value.defaultValue)).toBe('default"value'); }); + + it('handles strings with escaped unicode chars', async () => { + const dict = await getObject( + "const a = { key: 'key', defaultValue: 'default\\u{03A9}value' }", + FILE_NAME, + 'react' + ); + + JSON.stringify(dict.value.defaultValue); + + expect(extractValue(dict.value.key)).toBe('key'); + expect(extractValue(dict.value.defaultValue)).toBe( + 'default\u{03A9}value' + ); + }); }); }); }); From c8452c2cb0abfb13dab7ed535408768b752ad441 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0t=C4=9Bp=C3=A1n=20Gran=C3=A1t?= Date: Tue, 2 Jul 2024 15:26:30 +0200 Subject: [PATCH 3/3] fix: properly extract escaped characters and respect default namespace in magic comments --- src/test/unit/extractor/react/react.test.ts | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/test/unit/extractor/react/react.test.ts b/src/test/unit/extractor/react/react.test.ts index 51c8428..06531e2 100644 --- a/src/test/unit/extractor/react/react.test.ts +++ b/src/test/unit/extractor/react/react.test.ts @@ -1377,7 +1377,7 @@ describe('magic comments', () => { it('overrides data from code (JSX-specific)', async () => { const expected = [ - { keyName: 'key-override-1', namespace: 'namespace', line: 6 }, + { keyName: 'key-override-1', defaultValue: undefined, line: 6 }, ]; const code = ` @@ -1392,9 +1392,7 @@ describe('magic comments', () => { } `; - const extracted = await extractReactKeys(code, 'test.jsx', { - defaultNamespace: 'namespace', - }); + const extracted = await extractReactKeys(code, 'test.jsx'); expect(extracted.keys).toEqual(expected); expect(extracted.warnings).toEqual([]); });