fix: properly extract escaped characters and respect default namespace in magic comments

package.json

@@ -37,6 +37,7 @@
     "json5": "^2.2.3",
     "jsonschema": "^1.4.1",
     "openapi-fetch": "^0.9.7",
+    "unescape-js": "^1.1.4",
     "vscode-oniguruma": "^1.7.0",
     "vscode-textmate": "^9.0.0",
     "yauzl": "^2.10.0"

src/extractor/parser/generalMapper.ts

@@ -23,6 +23,8 @@ export const generalMapper = (token: Token) => {
     case 'string.quoted.single.ts':
     case 'string.quoted.double.ts':
       return 'string.body';
+    case 'constant.character.escape.ts':
+      return 'escaped.character';
 
     // template strings
     case 'punctuation.definition.string.template.begin.ts':

src/extractor/parser/generateReport.ts

@@ -37,7 +37,7 @@ function keyInfoFromComment(
     context.unusedComments.delete(commentAtLine);
     return {
       keyName: commentAtLine.keyName,
-      namespace: commentAtLine.namespace,
+      namespace: commentAtLine.namespace ?? context.options.defaultNamespace,
       defaultValue: commentAtLine.defaultValue,
       line: commentAtLine.line,
     };

src/extractor/parser/tokenMergers/stringMerger.ts

@@ -1,12 +1,11 @@
 import { GeneralTokenType } from '../generalMapper.js';
 import { MachineType } from '../mergerMachine.js';
+import unescape from 'unescape-js';
 
 export const enum S {
   Idle,
   RegularString,
-  RegularStringEnd,
   TemplateString,
-  TemplateStringEnd,
 }
 
 export const stringMerger = {
@@ -23,35 +22,39 @@ export const stringMerger = {
         break;
       case S.RegularString:
         if (type === 'string.body') {
-          return S.RegularStringEnd;
+          return S.RegularString;
+        } else if (type === 'escaped.character') {
+          return S.RegularString;
         } else if (type === 'string.end') {
           return end.MERGE_ALL;
         }
         break;
-      case S.RegularStringEnd:
-        if (type === 'string.end') {
-          return end.MERGE_ALL;
-        }
-        break;
       case S.TemplateString:
         if (type === 'string.template.body') {
-          return S.TemplateStringEnd;
+          return S.TemplateString;
+        } else if (type === 'escaped.character') {
+          return S.TemplateString;
         } else if (type === 'string.template.end') {
           return end.MERGE_ALL;
         }
         break;
-      case S.TemplateStringEnd:
-        if (type === 'string.template.end') {
-          return end.MERGE_ALL;
-        }
     }
   },
   customType: 'string',
   resultToken: (matched) => {
-    if (matched.length === 3) {
-      return matched[1].token;
-    } else {
-      return '';
-    }
+    const escaped = matched
+      .map((t) => {
+        switch (t.customType) {
+          case 'string.template.body':
+          case 'string.body':
+          case 'escaped.character':
+            return t.token;
+
+          default:
+            return '';
+        }
+      })
+      .join('');
+    return unescape(escaped);
   },
 } as const satisfies MachineType<GeneralTokenType, S>;

src/test/unit/extractor/shared/properties.test.ts

@@ -138,6 +138,99 @@ describe('Plain JavaScript', () => {
         expect(extractValue(dict.value.key)).toBe('key');
         expect(extractValue(dict.value.ns)).toBe('');
       });
+
+      it('handles strings with newlines', async () => {
+        const dict = await getObject(
+          "const a = { key: 'key', defaultValue: 'default\\nvalue' }",
+          FILE_NAME,
+          'react'
+        );
+
+        JSON.stringify(dict.value.defaultValue);
+
+        expect(extractValue(dict.value.key)).toBe('key');
+        expect(extractValue(dict.value.defaultValue)).toBe('default\nvalue');
+      });
+
+      it('handles strings with newlines in template strings', async () => {
+        const dict = await getObject(
+          "const a = { key: 'key', defaultValue: `default\\nvalue` }",
+          FILE_NAME,
+          'react'
+        );
+
+        JSON.stringify(dict.value.defaultValue);
+
+        expect(extractValue(dict.value.key)).toBe('key');
+        expect(extractValue(dict.value.defaultValue)).toBe('default\nvalue');
+      });
+
+      it('handles strings with escaped backslash', async () => {
+        const dict = await getObject(
+          "const a = { key: 'key', defaultValue: `default\\\\value` }",
+          FILE_NAME,
+          'react'
+        );
+
+        JSON.stringify(dict.value.defaultValue);
+
+        expect(extractValue(dict.value.key)).toBe('key');
+        expect(extractValue(dict.value.defaultValue)).toBe('default\\value');
+      });
+
+      it('handles strings with escaped tab', async () => {
+        const dict = await getObject(
+          "const a = { key: 'key', defaultValue: `default\\tvalue` }",
+          FILE_NAME,
+          'react'
+        );
+
+        JSON.stringify(dict.value.defaultValue);
+
+        expect(extractValue(dict.value.key)).toBe('key');
+        expect(extractValue(dict.value.defaultValue)).toBe('default\tvalue');
+      });
+
+      it('handles strings with escaped quotes', async () => {
+        const dict = await getObject(
+          "const a = { key: 'key', defaultValue: 'default\\'value' }",
+          FILE_NAME,
+          'react'
+        );
+
+        JSON.stringify(dict.value.defaultValue);
+
+        expect(extractValue(dict.value.key)).toBe('key');
+        expect(extractValue(dict.value.defaultValue)).toBe("default'value");
+      });
+
+      it('handles strings with escaped double quotes', async () => {
+        const dict = await getObject(
+          `const a = { key: "key", defaultValue: "default\\"value" }`,
+          FILE_NAME,
+          'react'
+        );
+
+        JSON.stringify(dict.value.defaultValue);
+
+        expect(extractValue(dict.value.key)).toBe('key');
+        expect(extractValue(dict.value.defaultValue)).toBe('default"value');
+      });
+
+      it('handles strings with escaped unicode chars', async () => {
+        const dict = await getObject(
+          "const a = { key: 'key', defaultValue: 'default\\u{03A9}value' }",
+          FILE_NAME,
+          'react'
+        );
+
+        JSON.stringify(dict.value.defaultValue);
+
+        expect(extractValue(dict.value.key)).toBe('key');
+        expect(extractValue(dict.value.defaultValue)).toBe(
+          'default\u{03A9}value'
+        );
+      });
     });
   });
 });

src/test/unit/extractor/shared/report.test.ts

@@ -64,5 +64,21 @@ describe('report not influenced by surrounding block', () => {
 
       expect(report.keys).toEqual([{ keyName: 'keyName', line: 3 }]);
     });
+
+    it('respects default namespace in magic comments', async () => {
+      const report = await getReport(
+        `
+        // @tolgee-key key-override
+        t('keyName')
+        `,
+        FILE_NAME,
+        'react',
+        { defaultNamespace: 'namespace' }
+      );
+
+      expect(report.keys).toEqual([
+        { keyName: 'key-override', namespace: 'namespace', line: 2 },
+      ]);
+    });
   });
 });