Also update generated patterns

src/tokenizer/python-token-patterns.g.ts

@@ -5,7 +5,7 @@
 // THIS FILE HAS BEEN GENERATED BY THE `syntax-to-token-pattern.py` GENERATOR
 // DO NOT EDIT THIS FILE DIRECTLY! INSTEAD RUN THE PYTHON SCRIPT.
 // ANY MANUAL EDITS MADE TO THIS FILE WILL BE OVERWRITTEN. YOU HAVE BEEN WARNED.
-// Last generated: 14/08/2024 16:34:44 (UTC+0)
+// Last generated: 15/08/2024 16:36:00 (UTC+0)
 
 import { MetaTokenType, KeywordTokenType, CharacterTokenType, EntityTokenType, OperatorTokenType, LiteralTokenType } from "./renpy-tokens";
 import { TokenPattern } from "./token-pattern-types";
@@ -2504,10 +2504,10 @@ export const stringQuotedSingleLine: TokenPattern = {
     debugName: "stringQuotedSingleLine",
 
     token: LiteralTokenType.String, /*string.quoted.single.python*/
-    begin: /(\b[rRuU])?(\b\w+)?(['"])/dg,
+    begin: /(?:\b([rR])(?=[uU]))?([uU])?(['"])/dg,
     beginCaptures: {
-        1: { token: MetaTokenType.StringStorageType, /*storage.type.string.python*/ },
-        2: { token: MetaTokenType.Invalid, /*invalid.illegal.prefix.python*/ },
+        1: { token: MetaTokenType.Invalid, /*invalid.illegal.prefix.python*/ },
+        2: { token: MetaTokenType.StringStorageType, /*storage.type.string.python*/ },
         3: { token: MetaTokenType.StringBegin, /*punctuation.definition.string.begin.python*/ },
     },
     // @ts-ignore: Back references in end patterns are replaced by begin matches at runtime

src/tokenizer/renpy-token-patterns.g.ts

@@ -5,7 +5,7 @@
 // THIS FILE HAS BEEN GENERATED BY THE `syntax-to-token-pattern.py` GENERATOR
 // DO NOT EDIT THIS FILE DIRECTLY! INSTEAD RUN THE PYTHON SCRIPT.
 // ANY MANUAL EDITS MADE TO THIS FILE WILL BE OVERWRITTEN. YOU HAVE BEEN WARNED.
-// Last generated: 14/08/2024 16:34:44 (UTC+0)
+// Last generated: 15/08/2024 16:36:00 (UTC+0)
 
 import { MetaTokenType, CharacterTokenType, LiteralTokenType, EntityTokenType, KeywordTokenType, EscapedCharacterTokenType, OperatorTokenType } from "./renpy-tokens";
 import { TokenPattern } from "./token-pattern-types";
@@ -332,14 +332,14 @@ export const keywords: TokenPattern = {
 
             // [TODO: Should probably only be a keyword in the expression] Renpy sub expression keywords
             token: KeywordTokenType.Other, /*keyword.other.renpy*/
-            match: /\b(?<!\.)(?:set|expression|at|with|from|style_prefix|spacing|textbutton)\b/g,
+            match: /\b(?<!\.)(?:set|expression|from|style_prefix|spacing|textbutton)\b/g,
         },
         {
             debugName: "keywords.patterns![4]",
 
             // [TODO: Should probably only be a keyword in the expression] Renpy sub expression keywords
             token: KeywordTokenType.Other, /*keyword.control.flow.other.renpy*/
-            match: /\b(?<!\.)(?:action)\b/g,
+            match: /\b(?<!\.)(?:action|at|with)\b/g,
         },
     ]
 };
@@ -403,26 +403,27 @@ export const constantPlaceholder: TokenPattern = {
 
     // Python value interpolation using [ ... ]
     token: MetaTokenType.ConstantCaps, /*constant.other.placeholder.tags.renpy*/
-    match: /(\[)(.*?)(\])(?![^\[]*?\])/dg,
-    captures: {
-        1: { token: MetaTokenType.Placeholder, /*constant.character.format.placeholder.other.renpy*/ },
-        2: {
-            token: MetaTokenType.PythonLine, /*meta.embedded.line.python*/
-            patterns: [
-                {
-                    debugName: "constantPlaceholder.captures![2].patterns![0]",
+    contentToken: MetaTokenType.PythonLine, /*meta.embedded.line.python*/
+    begin: /(?<=[^\[])\[/dg,
+    beginCaptures: {
+        0: { token: MetaTokenType.Placeholder, /*constant.character.format.placeholder.begin.renpy*/ },
+    },
+    end: /\]/dg,
+    endCaptures: {
+        0: { token: MetaTokenType.Placeholder, /*constant.character.format.placeholder.end.renpy*/ },
+    },
+    patterns: [
+        {
+            debugName: "constantPlaceholder.patterns![0]",
 
-                    token: MetaTokenType.InterpolateFlags, /*meta.interpolate.flags.renpy*/
-                    match: /(!)([rstiqulc]+)/dg,
-                    captures: {
-                        1: { token: CharacterTokenType.ExclamationMark, /*punctuation.definition.interpolate.flags.begin.renpy*/ },
-                        2: { token: MetaTokenType.ModifierFlagStorageType, /*storage.modifier.flag.renpy*/ },
-                    },
-                },
-            ]
+            token: MetaTokenType.InterpolateFlags, /*meta.interpolate.flags.renpy*/
+            match: /(!)([rstiqulc]+)/dg,
+            captures: {
+                1: { token: CharacterTokenType.ExclamationMark, /*punctuation.definition.interpolate.flags.begin.renpy*/ },
+                2: { token: MetaTokenType.ModifierFlagStorageType, /*storage.modifier.flag.renpy*/ },
+            },
         },
-        3: { token: MetaTokenType.Placeholder, /*constant.character.format.placeholder.other.renpy*/ },
-    },
+    ]
 };
 
 export const escapedUnicodeChar: TokenPattern = {
@@ -1089,7 +1090,7 @@ export const sayStatements: TokenPattern = {
             debugName: "sayStatements.patterns![1]",
 
             token: MetaTokenType.SayStatement, /*meta.say.narrator.renpy meta.say.statement.renpy*/
-            begin: /(?<=(^[ \t]+))(?=r?(?:"|'|`))/gm,
+            begin: /(?<=^[ \t]+)(?=r?(?:"|'|`))/gm,
             end: /$/gm,
             patterns: [
                 strings,
@@ -2047,6 +2048,7 @@ simpleExpression.patterns!.splice(7, 0, comments);
 stringsInterior.patterns!.splice(2, 0, stringTags);
 stringTags.patterns![10].captures![4].patterns![0].captures![3].patterns!.splice(0, 0, labelName);
 pythonStatements.patterns!.push(define, defaultStatement, oneLinePython, pythonBlockTester);
+sayStatements.patterns![1].patterns!.splice(2, 0, withStatement);
 renpyBlockTester.patterns!.push(basePatterns);
 layeredimageBlockTester.patterns!.push(layeredimageGroup, layeredimageAttribute, basePatterns);
 translateBlockTester.patterns![0].patterns!.push(basePatterns);

src/tokenizer/screen-token-patterns.g.ts

@@ -5,7 +5,7 @@
 // THIS FILE HAS BEEN GENERATED BY THE `syntax-to-token-pattern.py` GENERATOR
 // DO NOT EDIT THIS FILE DIRECTLY! INSTEAD RUN THE PYTHON SCRIPT.
 // ANY MANUAL EDITS MADE TO THIS FILE WILL BE OVERWRITTEN. YOU HAVE BEEN WARNED.
-// Last generated: 07/08/2024 16:42:09 (UTC+0)
+// Last generated: 15/08/2024 16:36:00 (UTC+0)
 
 import { EntityTokenType, CharacterTokenType, KeywordTokenType, MetaTokenType } from "./renpy-tokens";
 import { TokenPattern } from "./token-pattern-types";
@@ -54,13 +54,13 @@ export const screenBuildInProperties: TokenPattern = {
 
             // Special manipulation keywords
             token: MetaTokenType.ControlFlowKeyword, /*keyword.control.flow.renpy*/
-            match: /\b(?<!\.)(?:action|at|(?:un)?hovered|alt|value|dragged)\b/g,
+            match: /\b(?<!\.)(?:action|at|(?:un)?hovered|alt|value|dragged|released)\b/g,
         },
         {
             debugName: "screenBuildInProperties.patterns![2]",
 
             // Special manipulation keywords
-            match: /\b(?<!\.)(key|input|(?:text|image)?button|grid|auto|mousearea|side|timer|[vh]?bar|action|viewport|(?:vp)?grid|imagemap|hotspot|hotbar|transclude)\b/dg,
+            match: /\b(?<!\.)(key|default|input|(?:text|image)?button|grid|auto|mousearea|side|timer|[vh]?bar|action|viewport|(?:vp)?grid|imagemap|hotspot|hotbar|transclude)\b/dg,
             captures: {
                 1: { token: KeywordTokenType.Other, /*keyword.other.renpy*/ },
             },
@@ -231,13 +231,13 @@ export const screenKeywords: TokenPattern = {
             debugName: "screenKeywords.patterns![2]",
 
             token: KeywordTokenType.Other, /*keyword.other.renpy*/
-            match: /\b(?<!\.)(?:sensitive|tag|text|drag|draggroup|id|test)\b/g,
+            match: /\b(?<!\.)(?:sensitive|tag|text|drag|draggroup|id|test|style)\b[^\.]/g,
         },
         {
             debugName: "screenKeywords.patterns![3]",
 
             token: EntityTokenType.TagName, /*entity.other.attribute-name.transform.renpy entity.name.tag.css.transform.renpy*/
-            match: /\b(?<!\.)(?:modal|style|style_group|style_prefix|zorder|tooltip|variant|layer|roll_forward)\b/g,
+            match: /\b(?<!\.)(?:modal|style_group|style_prefix|zorder|tooltip|variant|layer|roll_forward)\b/g,
         },
         {
             debugName: "screenKeywords.patterns![4]",
@@ -254,12 +254,25 @@ export const screenBlockTester: TokenPattern = {
             debugName: "screenBlockTester.patterns![0]",
 
             contentToken: MetaTokenType.ScreenBlock, /*meta.screen.block.renpy*/
-            begin: /(?<=(^[ \t]*)(?:screen|frame|fixed|window|viewport|drag|draggroup|(?:vp)?grid|text|vbox|hbox|imagemap)\b.*?)(:)/dgm,
+            begin: /(?<=(^[ \t]*)(?:screen|frame|fixed|window|viewport|drag|draggroup|(?:vp)?grid|vbox|hbox|imagemap)\b.*?)(:)/dgm,
             beginCaptures: {
                 2: { token: CharacterTokenType.Colon, /*punctuation.section.screen.begin.renpy*/ },
             },
             // @ts-ignore: Back references in end patterns are replaced by begin matches at runtime
             end: /^(?=(?!\1)[ \t]*[^\s#]|\1[^\s#])/gm,
+            patterns: [
+            ]
+        },
+        {
+            debugName: "screenBlockTester.patterns![1]",
+
+            contentToken: MetaTokenType.ATLBlock, /*meta.atl.block.renpy*/
+            begin: /(?<=(^[ \t]*)(?:text)\b.*?)(:)/dgm,
+            beginCaptures: {
+                2: { token: CharacterTokenType.Colon, /*punctuation.section.atl.begin.renpy*/ },
+            },
+            // @ts-ignore: Back references in end patterns are replaced by begin matches at runtime
+            end: /^(?=(?!\1)[ \t]*[^\s#]|\1[^\s#])/gm,
             patterns: []
         }]
 };
@@ -297,51 +310,21 @@ export const screenWindow: TokenPattern = {
 export const screenText: TokenPattern = {
     debugName: "screenText",
 
-    contentToken: MetaTokenType.ScreenText, /*meta.screen.text.renpy*/
-    begin: /^([ \t]+)?(text)\b[ \t]*(:)/dgm,
+    token: MetaTokenType.ScreenText, /*meta.screen.text.renpy*/
+    begin: /(?<=^[ \t]*)(text)\b/dgm,
     beginCaptures: {
-        1: { token: CharacterTokenType.Whitespace, /*punctuation.whitespace.leading.block.renpy*/ },
-        2: { token: KeywordTokenType.Text, /*keyword.text.renpy*/ },
-        3: { token: CharacterTokenType.Colon, /*punctuation.section.block.begin.renpy*/ },
+        1: { token: KeywordTokenType.Text, /*keyword.text.renpy*/ },
     },
-    // @ts-ignore: Back references in end patterns are replaced by begin matches at runtime
-    end: /^(?=(?!\1)[ \t]*[^\s#]|\1[^\s#])/gm,
+    end: /(?=:)|$/gm,
     patterns: [
-        {
-            debugName: "screenText.patterns![0]",
-
-            token: MetaTokenType.ScreenText, /*meta.screen.text.renpy*/
-            contentToken: MetaTokenType.ATLBlock, /*meta.atl-block.renpy*/
-            begin: /^([ \t]+)?(text)\b[ \t]*([a-zA-Z_0-9 ]*)(:)/dgm,
-            beginCaptures: {
-                1: { token: CharacterTokenType.Whitespace, /*punctuation.whitespace.leading.renpy*/ },
-                2: { token: KeywordTokenType.Text, /*keyword.text.renpy*/ },
-                3: { token: EntityTokenType.TextName, /*entity.name.type.text.renpy*/ },
-                4: { token: CharacterTokenType.Colon, /*punctuation.section.atl.begin.renpy*/ },
-            },
-            // @ts-ignore: Back references in end patterns are replaced by begin matches at runtime
-            end: /^(?=(?!\1)[ \t]*[^\s#]|\1[^\s#])/gm,
-            patterns: [
-            ]
-        },
         {
             debugName: "screenText.patterns![1]",
 
-            token: MetaTokenType.ScreenText, /*meta.screen.text.renpy*/
-            begin: /(?<=^[ \t]*)(text)\b[ \t]*/dgm,
-            beginCaptures: {
-                1: { token: KeywordTokenType.Text, /*keyword.text.renpy*/ },
-            },
-            end: /(?=\b(at)\b|#|=)|$/gm,
-            patterns: [
-                {
-                    debugName: "screenText.patterns![1].patterns![1]",
-
-                    token: EntityTokenType.TextName, /*entity.name.type.text.renpy*/
-                    match: /\b(?:[a-zA-Z_0-9]+)\b[ \t]*/g,
-                },
-            ]
+            token: EntityTokenType.TextName, /*entity.name.type.text.renpy*/
+            match: /(?<=text[ \t]*)\b(?:[a-zA-Z_0-9]+)\b(?=[ \t]+|$)/gm,
         },
+        screenKeywords,
+        screenSimpleExpression,
     ]
 };
 
@@ -402,11 +385,13 @@ export const screenLanguage: TokenPattern = {
         screenWindow,
         screenBox,
         screenText,
+        screenBlockTester,
         screenKeywords,
         screenSimpleExpression,
         screenFallback,
     ]
 };
 
 // Push pattern references that were not defined on include
-screenBlockTester.patterns![0].patterns!.push(screenLanguage);
+screenBlockTester.patterns![0].patterns!.push(screenLanguage);
+screenBlockTester.patterns![1].patterns!.push(screenLanguage);

src/tokenizer/style-token-patterns.g.ts

@@ -5,11 +5,22 @@
 // THIS FILE HAS BEEN GENERATED BY THE `syntax-to-token-pattern.py` GENERATOR
 // DO NOT EDIT THIS FILE DIRECTLY! INSTEAD RUN THE PYTHON SCRIPT.
 // ANY MANUAL EDITS MADE TO THIS FILE WILL BE OVERWRITTEN. YOU HAVE BEEN WARNED.
-// Last generated: 14/08/2024 09:13:39 (UTC+0)
+// Last generated: 15/08/2024 16:36:00 (UTC+0)
 
 import { MetaTokenType, CharacterTokenType, EntityTokenType, KeywordTokenType } from "./renpy-tokens";
 import { TokenPattern } from "./token-pattern-types";
 
+export const builtinStyles: TokenPattern = {
+    patterns: [
+        {
+            debugName: "builtinStyles.patterns![0]",
+
+            token: MetaTokenType.BuiltinType, /*support.type.builtin.style.renpy*/
+            match: /\b(?<!\.)(?:(alt)?ruby_style)\b/g,
+        },
+    ]
+};
+
 export const styleBlockTester: TokenPattern = {
     patterns: [
         {
@@ -33,16 +44,22 @@ export const styleProperty: TokenPattern = {
 
             // ON EDIT: If you update this list, also update it in style-clause
             token: EntityTokenType.TagName, /*support.constant.property-key.test.renpy entity.name.tag.css.style.renpy*/
-            match: /\b(?<!\.)(?:(?:selected_)?(?:hover_|idle_|insensitive_|activate_)?(?:activate_sound|adjust_spacing|aft_bar|aft_gutter|alt|altruby_style|antialias|axis|background|bar_invert|bar_resizing|unscrollable|bar_vertical|black_color|bold|bottom_margin|bottom_padding|box_layout|box_reverse|box_wrap|box_wrap_spacing|caret|child|clipping|color|debug|drop_shadow|drop_shadow_color|emoji_font|extra_alt|first_indent|first_spacing|fit_first|focus_mask|focus_rect|font|fore_bar|fore_gutter|foreground|group_alt|hinting|hover_sound|hyperlink_functions|italic|instance|justify|kerning|key_events|keyboard_focus|language|layout|line_leading|left_margin|line_overlap_split|left_padding|line_spacing|mouse|modal|min_width|mipmap|newline_indent|order_reverse|outlines|outline_scaling|prefer_emoji|rest_indent|right_margin|right_padding|ruby_line_leading|ruby_style|shaper|size|size_group|slow_abortable|slow_cps|slow_cps_multiplier|spacing|strikethrough|subtitle_width|subpixel|text_y_fudge|text_align|thumb|thumb_offset|thumb_shadow|time_policy|top_margin|top_padding|underline|vertical|xanchor|xfill|xfit|xmaximum|xminimum|xoffset|xpos|xspacing|yanchor|yfill|yfit|ymaximum|yminimum|yoffset|ypos|yspacing|margin|xmargin|ymargin|xalign|yalign|padding|xpadding|ypadding|minwidth|textalign|slow_speed|enable_hover|left_gutter|right_gutter|top_gutter|bottom_gutter|left_bar|right_bar|top_bar|bottom_bar|base_bar|box_spacing|box_first_spacing|pos|anchor|offset|align|maximum|minimum|xsize|ysize|xysize|area|xcenter|ycenter|xycenter))\b/g,
+            match: /\b(?<!\.)(?:(?:selected_)?(?:hover_|idle_|insensitive_|activate_)?(?:activate_sound|adjust_spacing|aft_bar|aft_gutter|alt|antialias|axis|background|bar_invert|bar_resizing|unscrollable|bar_vertical|black_color|bold|bottom_margin|bottom_padding|box_layout|box_reverse|box_wrap|box_wrap_spacing|caret|child|clipping|color|debug|drop_shadow|drop_shadow_color|emoji_font|extra_alt|first_indent|first_spacing|fit_first|focus_mask|focus_rect|font|fore_bar|fore_gutter|foreground|group_alt|hinting|hover_sound|hyperlink_functions|italic|instance|justify|kerning|key_events|keyboard_focus|language|layout|line_leading|left_margin|line_overlap_split|left_padding|line_spacing|mouse|modal|min_width|mipmap|newline_indent|order_reverse|outlines|outline_scaling|prefer_emoji|rest_indent|right_margin|right_padding|ruby_line_leading|shaper|size|size_group|slow_abortable|slow_cps|slow_cps_multiplier|spacing|strikethrough|subtitle_width|subpixel|text_y_fudge|text_align|thumb|thumb_offset|thumb_shadow|time_policy|top_margin|top_padding|underline|vertical|xanchor|xfill|xfit|xmaximum|xminimum|xoffset|xpos|xspacing|yanchor|yfill|yfit|ymaximum|yminimum|yoffset|ypos|yspacing|margin|xmargin|ymargin|xalign|yalign|padding|xpadding|ypadding|minwidth|textalign|slow_speed|enable_hover|left_gutter|right_gutter|top_gutter|bottom_gutter|left_bar|right_bar|top_bar|bottom_bar|base_bar|box_spacing|box_first_spacing|pos|anchor|offset|align|maximum|minimum|xsize|ysize|xysize|area|xcenter|ycenter|xycenter))\b/g,
         },
         {
             debugName: "styleProperty.patterns![1]",
 
+            token: MetaTokenType.BuiltinType, /*support.type.builtin.style.renpy*/
+            match: /\b(?<!\.)(?:(alt)?ruby_style)\b/g,
+        },
+        {
+            debugName: "styleProperty.patterns![2]",
+
             token: EntityTokenType.TagName, /*support.constant.property-key.renpy entity.name.tag.css.style.renpy*/
             match: /\b(?<!\.)(?:properties)\b/g,
         },
         {
-            debugName: "styleProperty.patterns![2]",
+            debugName: "styleProperty.patterns![3]",
 
             // Any other tokens are invalid
             token: MetaTokenType.Invalid, /*invalid.illegal.style-property.name.renpy*/
@@ -133,13 +150,15 @@ export const style: TokenPattern = {
     },
     end: /$|^/gm,
     patterns: [
-        styleClause,
+        builtinStyles,
         {
             debugName: "style.patterns![1]",
 
+            // Only the first identifier is a style name
             token: EntityTokenType.StyleName, /*entity.name.type.style.renpy*/
-            match: /\b[\p{XID_Start}_]\p{XID_Continue}*\b/gu,
+            match: /(?<=\bstyle[ \t]*)\b[\p{XID_Start}_]\p{XID_Continue}*\b/gu,
         },
+        styleClause,
         styleBlockTester,
     ]
 };