feat: add angular extractor (#108)

tolgee · Oct 4, 2024 · ccc88aa · ccc88aa
1 parent 54e9f83
commit ccc88aa
Show file tree

Hide file tree

Showing 50 changed files with 5,242 additions and 3,689 deletions.
diff --git a/schema.json b/schema.json
@@ -33,7 +33,7 @@
     },
     "parser": {
       "description": "Override parser detection.",
-      "enum": ["react", "vue", "svelte"]
+      "enum": ["react", "vue", "svelte", "ngx"]
     },
     "push": {
       "type": "object",

diff --git a/scripts/grammars.ts b/scripts/grammars.ts
@@ -28,7 +28,8 @@ export const Grammars: Record<string, UrlString> = {
   Svelte:
     'https://raw.githubusercontent.com/sveltejs/language-tools/master/packages/svelte-vscode/syntaxes/svelte.tmLanguage.src.yaml',
   Vue: 'https://raw.githubusercontent.com/vuejs/language-tools/master/extensions/vscode/syntaxes/vue.tmLanguage.json',
-
+  AngularHtml:
+    'https://raw.githubusercontent.com/ghaschel/vscode-angular-html/master/syntaxes/html-template.ng.json',
   HTML: 'https://raw.githubusercontent.com/textmate/html.tmbundle/master/Syntaxes/HTML.plist',
 };
 
@@ -48,6 +49,11 @@ export const Licenses: LicenseInformation[] = [
     license:
       'https://raw.githubusercontent.com/vuejs/language-tools/master/extensions/vscode/LICENSE',
   },
+  {
+    grammars: ['AngularHtml'],
+    license:
+      'https://raw.githubusercontent.com/ghaschel/vscode-angular-html/master/LICENSE',
+  },
 ];
 
 // Transformers receive the grammar and a Record<keyof Grammars, string>, where the value is the downloaded TM grammar.

diff --git a/src/extractor/extractor.ts b/src/extractor/extractor.ts
@@ -4,6 +4,7 @@ import { Token } from './parser/types.js';
 import { tokensList } from './visualizers/printTokens.js';
 import { visualizeRules } from './visualizers/visualizeRules.js';
 import { ParserVue } from './parserVue/ParserVue.js';
+import { ParserNgx } from './parserNgx/ParserNgx.js';
 import { ParserSvelte } from './parserSvelte/ParserSvelte.js';
 import { ExtractOptions, ExtractionResult, ParserType } from './index.js';
 import { IteratorListener } from './parser/iterator.js';
@@ -16,6 +17,8 @@ function pickParser(format: ParserType) {
       return ParserVue();
     case 'svelte':
       return ParserSvelte();
+    case 'ngx':
+      return ParserNgx();
   }
 }
 
@@ -48,10 +51,16 @@ export async function extractTreeAndReport(
   });
 
   if (debug) {
-    console.log(JSON.stringify(result.tree, null, 2));
-    console.log(tokensList(tokensMerged));
-    console.log(visualizeRules(tokensMerged, code));
-    console.log(visualizeRules(tokensWithRules, code));
+    console.log(
+      JSON.stringify(result.tree, null, 2) +
+        '\n' +
+        tokensList(tokensMerged) +
+        '\n' +
+        visualizeRules(tokensMerged, code) +
+        '\n' +
+        visualizeRules(tokensWithRules, code) +
+        '\n'
+    );
   }
 
   return result;

diff --git a/src/extractor/index.ts b/src/extractor/index.ts
@@ -20,7 +20,7 @@ export type ExtractOptions = {
   defaultNamespace: string | undefined;
 };
 
-export type ParserType = 'react' | 'vue' | 'svelte';
+export type ParserType = 'react' | 'vue' | 'svelte' | 'ngx';
 
 export type Extractor = (
   fileContents: string,

diff --git a/src/extractor/parser/extractComment.ts b/src/extractor/parser/extractComment.ts
@@ -50,10 +50,6 @@ function isValidKeyOverride(data: any): data is KeyOverride {
   return true;
 }
 
-function getEndLine(token: TokenType) {
-  return token.line + (token.token.match(/\n/gm)?.length ?? 0);
-}
-
 type TokenType = {
   token: string;
   line: number;
@@ -62,12 +58,12 @@ type TokenType = {
 export function extractComment(
   token: TokenType
 ): MagicCommentEvent | undefined {
-  const comment = token.token.trim();
+  const comment = token.token.replaceAll(/[^\n]([\w]*\*+)/g, '').trim();
   if (comment.startsWith('@tolgee-ignore')) {
     return {
       type: 'MAGIC_COMMENT',
       kind: 'ignore',
-      line: getEndLine(token),
+      line: token.line,
     };
   }
 
@@ -80,7 +76,7 @@ export function extractComment(
         type: 'MAGIC_COMMENT',
         kind: 'key',
         keyName: data.slice(1),
-        line: getEndLine(token),
+        line: token.line,
       };
     }
 
@@ -93,7 +89,7 @@ export function extractComment(
           return {
             type: 'WARNING',
             kind: 'W_INVALID_KEY_OVERRIDE',
-            line: getEndLine(token),
+            line: token.line,
           };
         } else {
           return {
@@ -102,22 +98,22 @@ export function extractComment(
             keyName: key.key,
             namespace: key.ns,
             defaultValue: key.defaultValue,
-            line: getEndLine(token),
+            line: token.line,
           };
         }
       } catch {
         return {
           type: 'WARNING',
           kind: 'W_MALFORMED_KEY_OVERRIDE',
-          line: getEndLine(token),
+          line: token.line,
         };
       }
     }
     return {
       type: 'MAGIC_COMMENT',
       kind: 'key',
       keyName: data,
-      line: getEndLine(token),
+      line: token.line,
     };
   }
 }
diff --git a/src/extractor/parser/generalMapper.ts b/src/extractor/parser/generalMapper.ts
@@ -17,9 +17,9 @@ export const generalMapper = (token: Token) => {
 
     // plain strings
     case 'punctuation.definition.string.begin.ts':
-      return 'string.begin';
+      return 'string.quote';
     case 'punctuation.definition.string.end.ts':
-      return 'string.end';
+      return 'string.quote';
     case 'string.quoted.single.ts':
     case 'string.quoted.double.ts':
       return 'string.body';
@@ -28,9 +28,9 @@ export const generalMapper = (token: Token) => {
 
     // template strings
     case 'punctuation.definition.string.template.begin.ts':
-      return 'string.teplate.begin';
+      return 'string.teplate.quote';
     case 'punctuation.definition.string.template.end.ts':
-      return 'string.template.end';
+      return 'string.template.quote';
     case 'string.template.ts':
       return 'string.template.body';
 

diff --git a/src/extractor/parser/mergerMachine.ts b/src/extractor/parser/mergerMachine.ts
@@ -1,5 +1,11 @@
 import { Token } from './types.js';
 
+export type MergeResult<T extends string | undefined> = {
+  before: Token<T>[];
+  toMerge: Token<T>[];
+  after: Token<T>[];
+};
+
 export type MachineType<
   TokenType extends string | undefined,
   State,
@@ -12,34 +18,37 @@ export type MachineType<
     end: typeof endOptions
   ) => State | symbol | undefined;
   customType: CustomTokenType;
-  resultToken?: (matched: Token<TokenType>[]) => string;
+  resultToken?: (matched: Token<TokenType>[]) => Partial<Token>;
+  customMerge?: (tokens: Token<TokenType>[]) => MergeResult<TokenType>;
 };
 
-function defaultResultToken<T extends string | undefined>(matched: Token<T>[]) {
-  return matched.map((t) => t.token).join('');
-}
-
 const MERGE_ALL = Symbol('MERGE_ALL');
 const MERGE_WITHOUT_LAST = Symbol('MERGE_WITHOUT_LAST');
+const REPLACE_FIRST = Symbol('REPLACE_FIRST');
+const MERGE_CUSTOM = Symbol('MERGE_CUSTOM');
 
 export const endOptions = {
   MERGE_ALL,
   MERGE_WITHOUT_LAST,
+  REPLACE_FIRST,
+  MERGE_CUSTOM,
 };
 
 function createNewToken(
   tokens: Token[],
   customType: string,
-  merger: (matched: Token[]) => string
+  merger: ((matched: Token[]) => Partial<Token>) | undefined
 ) {
+  const mergerData = merger?.(tokens);
   return {
     customType,
     type: 'custom',
     startIndex: tokens[0].startIndex,
     endIndex: tokens[tokens.length - 1].endIndex,
     scopes: [],
     line: tokens[0].line,
-    token: merger(tokens),
+    token: tokens.map((t) => t.token).join(''),
+    ...mergerData,
   } satisfies Token;
 }
 
@@ -72,21 +81,45 @@ export function createMachine<M extends MachineType<any, any, any>>(
         continue;
       } else if (
         newState === endOptions.MERGE_ALL ||
-        newState === endOptions.MERGE_WITHOUT_LAST
+        newState === endOptions.MERGE_WITHOUT_LAST ||
+        newState === endOptions.REPLACE_FIRST ||
+        newState === endOptions.MERGE_CUSTOM
       ) {
-        const lastToken =
-          newState === endOptions.MERGE_WITHOUT_LAST ? stack.pop() : undefined;
+        let before: Token[] = [];
+        let toMerge: Token[];
+        let after: Token[];
+        if (newState === endOptions.MERGE_ALL) {
+          toMerge = stack;
+          after = [];
+        } else if (newState === endOptions.MERGE_WITHOUT_LAST) {
+          after = [stack.pop()!];
+          toMerge = stack;
+        } else if (newState === endOptions.REPLACE_FIRST) {
+          toMerge = [stack.shift()!];
+          after = stack;
+        } else {
+          if (!machine.customMerge) {
+            throw new Error('No custom merge cpecified');
+          }
+          const result = machine.customMerge(stack);
+          before = result.before;
+          toMerge = result.toMerge;
+          after = result.after;
+        }
         const newToken = createNewToken(
-          stack,
+          toMerge,
           machine.customType,
-          machine.resultToken ?? defaultResultToken
+          machine.resultToken
         );
-        state = machine.initial;
-        stack = [];
+        for (const result of before) {
+          yield result;
+        }
         yield newToken;
-        if (lastToken) {
-          yield lastToken;
+        for (const result of after) {
+          yield result;
         }
+        stack = [];
+        state = machine.initial;
         continue;
       } else {
         state = newState;

diff --git a/src/extractor/parser/tokenMergers/closingTagMerger.ts b/src/extractor/parser/tokenMergers/closingTagMerger.ts
@@ -31,6 +31,6 @@ export const closingTagMerger = {
   },
   customType: 'tag.closing',
   resultToken: (matched) => {
-    return matched[1].token;
+    return { token: matched[1].token };
   },
 } as const satisfies MachineType<ReactMappedTokenType, S>;
diff --git a/src/extractor/parser/tokenMergers/commentsMerger.ts b/src/extractor/parser/tokenMergers/commentsMerger.ts
@@ -21,7 +21,7 @@ export const commentsMerger = {
       case S.CommentStart:
         if (type === 'comment.line') {
           return end.MERGE_ALL;
-        } else if (type === 'comment.block') {
+        } else if (type === 'comment.block' || token.token === '\n') {
           return S.CommentBlock;
         }
         break;
@@ -36,10 +36,13 @@ export const commentsMerger = {
     }
   },
   resultToken: (matched) => {
-    return matched
-      .filter((t) => t.customType && t.customType !== 'comment.definition')
-      .map((t) => t.token)
-      .join('');
+    return {
+      token: matched
+        .filter((t) => t.customType && t.customType !== 'comment.definition')
+        .map((t) => t.token)
+        .join(''),
+      line: matched[matched.length - 1].line,
+    };
   },
   customType: 'comment',
 } as const satisfies MachineType<GeneralTokenType, S>;
diff --git a/src/extractor/parser/tokenMergers/stringMerger.ts b/src/extractor/parser/tokenMergers/stringMerger.ts
@@ -14,9 +14,9 @@ export const stringMerger = {
     const type = token.customType;
     switch (state) {
       case S.Idle:
-        if (type === 'string.begin') {
+        if (type === 'string.quote') {
           return S.RegularString;
-        } else if (type === 'string.teplate.begin') {
+        } else if (type === 'string.teplate.quote') {
           return S.TemplateString;
         }
         break;
@@ -25,7 +25,7 @@ export const stringMerger = {
           return S.RegularString;
         } else if (type === 'escaped.character') {
           return S.RegularString;
-        } else if (type === 'string.end') {
+        } else if (type === 'string.quote') {
           return end.MERGE_ALL;
         }
         break;
@@ -34,7 +34,7 @@ export const stringMerger = {
           return S.TemplateString;
         } else if (type === 'escaped.character') {
           return S.TemplateString;
-        } else if (type === 'string.template.end') {
+        } else if (type === 'string.template.quote') {
           return end.MERGE_ALL;
         }
         break;
@@ -55,6 +55,6 @@ export const stringMerger = {
         }
       })
       .join('');
-    return unescape(escaped);
+    return { token: unescape(escaped) };
   },
 } as const satisfies MachineType<GeneralTokenType, S>;
diff --git a/src/extractor/parser/tokenMergers/templateStringMerger.ts b/src/extractor/parser/tokenMergers/templateStringMerger.ts
@@ -60,6 +60,6 @@ export const templateStringMerger = {
       }
     }
 
-    return result.join('').trim();
+    return { token: result.join('').trim() };
   },
 } as const satisfies MachineType<ReactMappedTokenType, S>;
diff --git a/src/extractor/parser/tree/getTranslateProps.ts b/src/extractor/parser/tree/getTranslateProps.ts
@@ -1,6 +1,6 @@
 import { ArrayNode, DictNode, GeneralNode } from '../types.js';
 
-function getCombinedOptions(
+export function getCombinedOptions(
   {
     ns,
     noWrap,