diff --git a/packages/core/src/constructors/bundle-factory.ts b/packages/core/src/constructors/bundle-factory.ts index ac0914dde..a1ccedaca 100644 --- a/packages/core/src/constructors/bundle-factory.ts +++ b/packages/core/src/constructors/bundle-factory.ts @@ -190,7 +190,9 @@ export interface ShorthandsBundle { * Shorthand for `getLastGrammarState` with auto-loaded theme and language. * A singleton highlighter it maintained internally. */ - getLastGrammarState: (code: string, options: CodeToTokensBaseOptions) => Promise + getLastGrammarState: + | ((element: ThemedToken[][] | Root) => GrammarState) + | ((code: string, options: CodeToTokensBaseOptions) => Promise) } export function makeSingletonHighlighter( diff --git a/packages/core/src/constructors/highlighter.ts b/packages/core/src/constructors/highlighter.ts index c9f5adb48..198216080 100644 --- a/packages/core/src/constructors/highlighter.ts +++ b/packages/core/src/constructors/highlighter.ts @@ -20,7 +20,7 @@ export async function createHighlighterCore(options: HighlighterCoreOptions = {} const internal = await createShikiInternal(options) return { - getLastGrammarState: (code, options) => getLastGrammarState(internal, code, options), + getLastGrammarState: (...args: any[]) => getLastGrammarState(internal, ...args as [any])!, codeToTokensBase: (code, options) => codeToTokensBase(internal, code, options), codeToTokensWithThemes: (code, options) => codeToTokensWithThemes(internal, code, options), codeToTokens: (code, options) => codeToTokens(internal, code, options), @@ -43,7 +43,7 @@ export function createHighlighterCoreSync(options: HighlighterCoreOptions const internal = createShikiInternalSync(options) return { - getLastGrammarState: (code, options) => getLastGrammarState(internal, code, options), + getLastGrammarState: (...args: any[]) => getLastGrammarState(internal, ...args as [any, any]), codeToTokensBase: (code, options) => codeToTokensBase(internal, code, options), codeToTokensWithThemes: (code, options) => codeToTokensWithThemes(internal, code, options), codeToTokens: (code, options) => codeToTokens(internal, code, options), diff --git a/packages/core/src/highlight/code-to-hast.ts b/packages/core/src/highlight/code-to-hast.ts index 4d2bfa921..abe65be57 100644 --- a/packages/core/src/highlight/code-to-hast.ts +++ b/packages/core/src/highlight/code-to-hast.ts @@ -1,6 +1,7 @@ import type { CodeToHastOptions, CodeToHastRenderOptions, + GrammarState, ShikiInternal, ShikiTransformerContext, ShikiTransformerContextCommon, @@ -14,7 +15,7 @@ import type { } from 'hast' import { FontStyle } from '@shikijs/vscode-textmate' - +import { getLastGrammarStateFromMap, setLastGrammarStateToMap } from '../textmate/grammar-state' import { addClassToHast, getTokenStyleObject, stringifyTokenStyle } from '../utils' import { warnDeprecated } from '../warn' import { getTransformers } from './_get-transformers' @@ -42,6 +43,7 @@ export function codeToHast( bg, themeName, rootStyle, + grammarState, } = codeToTokens(internal, input, options) const { @@ -73,6 +75,7 @@ export function codeToHast( rootStyle, }, contextSource, + grammarState, ) } @@ -80,6 +83,7 @@ export function tokensToHast( tokens: ThemedToken[][], options: CodeToHastRenderOptions, transformerContext: ShikiTransformerContextSource, + grammarState: GrammarState | undefined = getLastGrammarStateFromMap(tokens), ): Root { const transformers = getTransformers(options) @@ -220,6 +224,9 @@ export function tokensToHast( for (const transformer of transformers) result = transformer?.root?.call(context, result) || result + if (grammarState) + setLastGrammarStateToMap(result, grammarState) + return result } diff --git a/packages/core/src/highlight/code-to-tokens-base.ts b/packages/core/src/highlight/code-to-tokens-base.ts index 430ef1b9b..18da0b335 100644 --- a/packages/core/src/highlight/code-to-tokens-base.ts +++ b/packages/core/src/highlight/code-to-tokens-base.ts @@ -3,6 +3,8 @@ *-------------------------------------------------------- */ import type { CodeToTokensBaseOptions, + Grammar, + GrammarState, ShikiInternal, ThemedToken, ThemedTokenScopeExplanation, @@ -11,15 +13,15 @@ import type { } from '@shikijs/types' import type { FontStyle, - IGrammar, IRawThemeSetting, StateStack, } from '@shikijs/vscode-textmate' +import type { Root } from 'hast' import { ShikiError } from '@shikijs/types' -import { EncodedTokenMetadata, INITIAL } from '@shikijs/vscode-textmate' -import { getGrammarStack, GrammarState } from '../textmate/grammar-state' +import { EncodedTokenMetadata, INITIAL } from '@shikijs/vscode-textmate' +import { getGrammarStack, getLastGrammarStateFromMap, GrammarState as GrammarStateImpl, setLastGrammarStateToMap } from '../textmate/grammar-state' import { applyColorReplacements, isNoneTheme, isPlainLang, resolveColorReplacements, splitLines } from '../utils' import { tokenizeAnsiWithTheme } from './code-to-tokens-ansi' @@ -50,19 +52,29 @@ export function codeToTokensBase( if (options.grammarState.lang !== _grammar.name) { throw new ShikiError(`Grammar state language "${options.grammarState.lang}" does not match highlight language "${_grammar.name}"`) } - if (options.grammarState.theme !== themeName) { - throw new ShikiError(`Grammar state theme "${options.grammarState.theme}" does not match highlight theme "${themeName}"`) + if (!options.grammarState.themes.includes(theme.name)) { + throw new ShikiError(`Grammar state themes "${options.grammarState.themes}" do not contain highlight theme "${theme.name}"`) } } return tokenizeWithTheme(code, _grammar, theme, colorMap, options) } +export function getLastGrammarState( + internal: ShikiInternal, + element: ThemedToken[][] | Root +): GrammarState | undefined export function getLastGrammarState( internal: ShikiInternal, code: string, - options: CodeToTokensBaseOptions = {}, -): GrammarState { + options?: CodeToTokensBaseOptions +): GrammarState +export function getLastGrammarState(...args: any[]): GrammarState | undefined { + if (args.length === 2) { + return getLastGrammarStateFromMap(args[1]) + } + + const [internal, code, options = {}] = args as [ShikiInternal, string, CodeToTokensBaseOptions] const { lang = 'text', theme: themeName = internal.getLoadedThemes()[0], @@ -77,7 +89,7 @@ export function getLastGrammarState( const _grammar = internal.getLanguage(lang) - return new GrammarState( + return new GrammarStateImpl( _tokenizeWithTheme(code, _grammar, theme, colorMap, options).stateStack, _grammar.name, theme.name, @@ -92,17 +104,27 @@ interface ThemeSettingsSelectors { export function tokenizeWithTheme( code: string, - grammar: IGrammar, + grammar: Grammar, theme: ThemeRegistrationResolved, colorMap: string[], options: TokenizeWithThemeOptions, ): ThemedToken[][] { - return _tokenizeWithTheme(code, grammar, theme, colorMap, options).tokens + const result = _tokenizeWithTheme(code, grammar, theme, colorMap, options) + + const grammarState = new GrammarStateImpl( + _tokenizeWithTheme(code, grammar, theme, colorMap, options).stateStack, + grammar.name, + theme.name, + ) + + setLastGrammarStateToMap(result.tokens, grammarState) + + return result.tokens } function _tokenizeWithTheme( code: string, - grammar: IGrammar, + grammar: Grammar, theme: ThemeRegistrationResolved, colorMap: string[], options: TokenizeWithThemeOptions, @@ -120,7 +142,7 @@ function _tokenizeWithTheme( const lines = splitLines(code) let stateStack = options.grammarState - ? getGrammarStack(options.grammarState) + ? getGrammarStack(options.grammarState, theme.name) ?? INITIAL : options.grammarContextCode != null ? _tokenizeWithTheme( options.grammarContextCode, diff --git a/packages/core/src/highlight/code-to-tokens-themes.ts b/packages/core/src/highlight/code-to-tokens-themes.ts index 1c2a48d60..8e955066e 100644 --- a/packages/core/src/highlight/code-to-tokens-themes.ts +++ b/packages/core/src/highlight/code-to-tokens-themes.ts @@ -4,6 +4,7 @@ import type { ThemedToken, ThemedTokenWithVariants, } from '@shikijs/types' +import { getLastGrammarStateFromMap, GrammarState, setLastGrammarStateToMap } from '../textmate/grammar-state' import { codeToTokensBase } from './code-to-tokens-base' /** @@ -19,11 +20,24 @@ export function codeToTokensWithThemes( .filter(i => i[1]) .map(i => ({ color: i[0], theme: i[1]! })) - const tokens = syncThemesTokenization( - ...themes.map(t => codeToTokensBase(internal, code, { + const themedTokens = themes.map((t) => { + const tokens = codeToTokensBase(internal, code, { ...options, theme: t.theme, - })), + }) + const state = getLastGrammarStateFromMap(tokens) + const theme = typeof t.theme === 'string' + ? t.theme + : t.theme.name + return { + tokens, + state, + theme, + } + }) + + const tokens = syncThemesTokenization( + ...themedTokens.map(i => i.tokens), ) const mergedTokens: ThemedTokenWithVariants[][] = tokens[0] @@ -54,6 +68,15 @@ export function codeToTokensWithThemes( }), ) + const mergedGrammarState = themedTokens[0].state + ? new GrammarState( + Object.fromEntries(themedTokens.map(s => [s.theme, s.state?.getInternalStack(s.theme)])), + themedTokens[0].state.lang, + ) + : undefined + if (mergedGrammarState) + setLastGrammarStateToMap(mergedTokens, mergedGrammarState) + return mergedTokens } diff --git a/packages/core/src/highlight/code-to-tokens.ts b/packages/core/src/highlight/code-to-tokens.ts index ef794dd44..417d6bc6f 100644 --- a/packages/core/src/highlight/code-to-tokens.ts +++ b/packages/core/src/highlight/code-to-tokens.ts @@ -1,5 +1,6 @@ -import type { CodeToTokensOptions, ShikiInternal, ThemedToken, ThemedTokenWithVariants, TokensResult } from '@shikijs/types' -import { ShikiError } from '../../../types/src/error' +import type { CodeToTokensOptions, GrammarState, ShikiInternal, ThemedToken, ThemedTokenWithVariants, TokensResult } from '@shikijs/types' +import { ShikiError } from '@shikijs/types' +import { getLastGrammarStateFromMap, setLastGrammarStateToMap } from '../textmate/grammar-state' import { applyColorReplacements, getTokenStyleObject, resolveColorReplacements } from '../utils' import { codeToTokensBase } from './code-to-tokens-base' import { codeToTokensWithThemes } from './code-to-tokens-themes' @@ -19,6 +20,7 @@ export function codeToTokens( let tokens: ThemedToken[][] let themeName: string let rootStyle: string | undefined + let grammarState: GrammarState | undefined if ('themes' in options) { const { @@ -41,6 +43,8 @@ export function codeToTokens( options, ) + grammarState = getLastGrammarStateFromMap(themeTokens) + if (defaultColor && !themes.find(t => t.color === defaultColor)) throw new ShikiError(`\`themes\` option must contain the defaultColor key \`${defaultColor}\``) @@ -49,6 +53,9 @@ export function codeToTokens( tokens = themeTokens .map(line => line.map(token => mergeToken(token, themesOrder, cssVariablePrefix, defaultColor))) + if (grammarState) + setLastGrammarStateToMap(tokens, grammarState) + const themeColorReplacements = themes.map(t => resolveColorReplacements(t.theme, options)) fg = themes.map((t, idx) => (idx === 0 && defaultColor @@ -73,6 +80,7 @@ export function codeToTokens( bg = applyColorReplacements(_theme.bg, colorReplacements) fg = applyColorReplacements(_theme.fg, colorReplacements) themeName = _theme.name + grammarState = getLastGrammarStateFromMap(tokens) } else { throw new ShikiError('Invalid options, either `theme` or `themes` must be provided') @@ -84,6 +92,7 @@ export function codeToTokens( bg, themeName, rootStyle, + grammarState, } } diff --git a/packages/core/src/textmate/grammar-state.ts b/packages/core/src/textmate/grammar-state.ts index 79080e3af..10e848da0 100644 --- a/packages/core/src/textmate/grammar-state.ts +++ b/packages/core/src/textmate/grammar-state.ts @@ -1,8 +1,24 @@ -import type { GrammarState as GrammarStateInterface } from '@shikijs/types' +import type { GrammarState as GrammarStateInterface, GrammarStateMapKey } from '@shikijs/types' import type { StateStack, StateStackImpl } from '@shikijs/vscode-textmate' import { INITIAL } from '@shikijs/vscode-textmate' import { ShikiError } from '../../../types/src/error' +import { toArray } from '../utils' + +const _grammarStateMap = new WeakMap() + +export function setLastGrammarStateToMap( + keys: GrammarStateMapKey, + state: GrammarStateInterface, +): void { + _grammarStateMap.set(keys, state) +} + +export function getLastGrammarStateFromMap( + keys: GrammarStateMapKey, +): GrammarStateInterface | undefined { + return _grammarStateMap.get(keys) +} /** * GrammarState is a special reference object that holds the state of a grammar. @@ -10,31 +26,85 @@ import { ShikiError } from '../../../types/src/error' * It's used to highlight code snippets that are part of the target language. */ export class GrammarState implements GrammarStateInterface { + /** + * Theme to Stack mapping + */ + private _stacks: Record = {} + public readonly lang: string + + get themes(): string[] { + return Object.keys(this._stacks) + } + + get theme(): string { + return this.themes[0] + } + + private get _stack(): StateStack { + return this._stacks[this.theme] + } + /** * Static method to create a initial grammar state. */ - static initial(lang: string, theme: string): GrammarState { - return new GrammarState(INITIAL, lang, theme) + static initial(lang: string, themes: string | string[]): GrammarState { + return new GrammarState( + Object.fromEntries(toArray(themes).map(theme => [theme, INITIAL])), + lang, + ) } constructor( - private readonly _stack: StateStack, - public readonly lang: string, - public readonly theme: string, - ) {} + stack: StateStack, + lang: string, + theme: string, + ) + constructor( + stacksMap: Record, + lang: string, + ) + constructor(...args: any[]) { + if (args.length === 2) { + const [stacksMap, lang] = args as [Record, string] + this.lang = lang + this._stacks = stacksMap + } + else { + const [stack, lang, theme] = args as [StateStack, string, string] + this.lang = lang + this._stacks = { [theme]: stack } + } + } + /** + * Get the internal stack object. + * @internal + */ + getInternalStack(theme = this.theme): StateStack | undefined { + return this._stacks[theme] + } + + /** + * @deprecated use `getScopes` instead + */ get scopes(): string[] { - return getScopes(this._stack as StateStackImpl) + return getScopes(this._stacks[this.theme] as StateStackImpl) + } + + getScopes(theme: string = this.theme): string[] { + return getScopes(this._stacks[theme] as StateStackImpl) } toJSON(): { lang: string theme: string + themes: string[] scopes: string[] } { return { lang: this.lang, theme: this.theme, + themes: this.themes, scopes: this.scopes, } } @@ -59,9 +129,11 @@ function getScopes(stack: StateStackImpl): string[] { return scopes } -export function getGrammarStack(state: GrammarState | GrammarStateInterface): StateStack { +export function getGrammarStack( + state: GrammarState | GrammarStateInterface, + theme?: string, +): StateStack | undefined { if (!(state instanceof GrammarState)) throw new ShikiError('Invalid grammar state') - // @ts-expect-error _stack is private - return state._stack + return state.getInternalStack(theme) } diff --git a/packages/engine-javascript/test/__records__/html-basic.json b/packages/engine-javascript/test/__records__/html-basic.json index af26b4732..acc52f8ad 100644 --- a/packages/engine-javascript/test/__records__/html-basic.json +++ b/packages/engine-javascript/test/__records__/html-basic.json @@ -200,6 +200,88 @@ ] } }, + { + "args": [ + "
bar
\n", + 0 + ], + "result": { + "index": 35, + "captureIndices": [ + { + "start": 0, + "end": 0, + "length": 0 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + }, + { + "start": 0, + "end": 1, + "length": 1 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + }, + { + "start": 1, + "end": 4, + "length": 3 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + }, + { + "start": 4, + "end": 5, + "length": 1 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + }, + { + "start": 4, + "end": 5, + "length": 1 + } + ] + } + }, + { + "args": [ + "foobar\n", + 0 + ], + "result": { + "index": 77, + "captureIndices": [ + { + "start": 0, + "end": 1, + "length": 1 + } + ] + } + }, { "args": [ "foobar\n", @@ -282,6 +364,118 @@ ] } }, + { + "args": [ + "
bar
\n", + 26 + ], + "result": { + "index": 0, + "captureIndices": [ + { + "start": 26, + "end": 26, + "length": 0 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + } + ] + } + }, + { + "args": [ + "
bar
\n", + 0 + ], + "result": { + "index": 1, + "captureIndices": [ + { + "start": 0, + "end": 0, + "length": 0 + }, + { + "start": 0, + "end": 1, + "length": 1 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + }, + { + "start": 1, + "end": 4, + "length": 3 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + }, + { + "start": 4, + "end": 5, + "length": 1 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + }, + { + "start": 4, + "end": 5, + "length": 1 + } + ] + } + }, { "args": [ "
bar
\n", @@ -476,29 +670,55 @@ } ] } - } - ] - }, - { - "constractor": [ - [ - "(?=[/]?>)", - "/\\*\\*(?!/)", - "(/\\*)(?:\\s*((@)internal)(?=\\s|(\\*/)))?", - "(^[ \\t]+)?((//)(?:\\s*((@)internal)(?=\\s|$))?)", - "<", - "\\s+" - ] - ], - "executions": [ + }, { "args": [ "
bar
\n", - 4 + 0 ], "result": { - "index": 5, + "index": 1, "captureIndices": [ + { + "start": 0, + "end": 4, + "length": 4 + }, + { + "start": 0, + "end": 1, + "length": 1 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + }, + { + "start": 1, + "end": 4, + "length": 3 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + }, + { + "start": 4, + "end": 5, + "length": 1 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + }, { "start": 4, "end": 5, @@ -513,7 +733,141 @@ 16 ], "result": { - "index": 0, + "index": 2, + "captureIndices": [ + { + "start": 16, + "end": 17, + "length": 1 + }, + { + "start": 16, + "end": 17, + "length": 1 + } + ] + } + }, + { + "args": [ + "
bar
\n", + 20 + ], + "result": { + "index": 0, + "captureIndices": [ + { + "start": 20, + "end": 26, + "length": 6 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + }, + { + "start": 20, + "end": 22, + "length": 2 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + }, + { + "start": 22, + "end": 25, + "length": 3 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + }, + { + "start": 25, + "end": 26, + "length": 1 + } + ] + } + } + ] + }, + { + "constractor": [ + [ + "(?=[/]?>)", + "/\\*\\*(?!/)", + "(/\\*)(?:\\s*((@)internal)(?=\\s|(\\*/)))?", + "(^[ \\t]+)?((//)(?:\\s*((@)internal)(?=\\s|$))?)", + "<", + "\\s+" + ] + ], + "executions": [ + { + "args": [ + "
bar
\n", + 4 + ], + "result": { + "index": 5, + "captureIndices": [ + { + "start": 4, + "end": 5, + "length": 1 + } + ] + } + }, + { + "args": [ + "
bar
\n", + 16 + ], + "result": { + "index": 0, + "captureIndices": [ + { + "start": 16, + "end": 16, + "length": 0 + } + ] + } + }, + { + "args": [ + "
bar
\n", + 4 + ], + "result": { + "index": 5, + "captureIndices": [ + { + "start": 4, + "end": 5, + "length": 1 + } + ] + } + }, + { + "args": [ + "
bar
\n", + 16 + ], + "result": { + "index": 0, "captureIndices": [ { "start": 16, @@ -541,6 +895,85 @@ ] ], "executions": [ + { + "args": [ + "
bar
\n", + 5 + ], + "result": { + "index": 4, + "captureIndices": [ + { + "start": 5, + "end": 10, + "length": 5 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + }, + { + "start": 5, + "end": 10, + "length": 5 + } + ] + } + }, + { + "args": [ + "
bar
\n", + 10 + ], + "result": { + "index": 5, + "captureIndices": [ + { + "start": 10, + "end": 11, + "length": 1 + } + ] + } + }, + { + "args": [ + "
bar
\n", + 11 + ], + "result": { + "index": 6, + "captureIndices": [ + { + "start": 11, + "end": 12, + "length": 1 + } + ] + } + }, + { + "args": [ + "
bar
\n", + 16 + ], + "result": { + "index": 0, + "captureIndices": [ + { + "start": 16, + "end": 16, + "length": 0 + } + ] + } + }, { "args": [ "
bar
\n", @@ -630,6 +1063,22 @@ ] ], "executions": [ + { + "args": [ + "
bar
\n", + 12 + ], + "result": { + "index": 0, + "captureIndices": [ + { + "start": 15, + "end": 16, + "length": 1 + } + ] + } + }, { "args": [ "
bar
\n", @@ -659,6 +1108,22 @@ ] ], "executions": [ + { + "args": [ + "
bar
\n", + 17 + ], + "result": { + "index": 0, + "captureIndices": [ + { + "start": 20, + "end": 20, + "length": 0 + } + ] + } + }, { "args": [ "
bar
\n", @@ -819,6 +1284,13 @@ ], "result": null }, + { + "args": [ + "
bar
\n", + 26 + ], + "result": null + }, { "args": [ "foobar\n", @@ -940,10 +1412,132 @@ 75 ], "result": null - } - ] - }, - { + }, + { + "args": [ + "foobar\n", + 1 + ], + "result": { + "index": 79, + "captureIndices": [ + { + "start": 1, + "end": 2, + "length": 1 + } + ] + } + }, + { + "args": [ + "foobar\n", + 2 + ], + "result": { + "index": 115, + "captureIndices": [ + { + "start": 2, + "end": 9, + "length": 7 + }, + { + "start": 2, + "end": 9, + "length": 7 + } + ] + } + }, + { + "args": [ + "foobar\n", + 9 + ], + "result": { + "index": 116, + "captureIndices": [ + { + "start": 10, + "end": 14, + "length": 4 + } + ] + } + }, + { + "args": [ + "foobar\n", + 14 + ], + "result": { + "index": 77, + "captureIndices": [ + { + "start": 14, + "end": 15, + "length": 1 + } + ] + } + }, + { + "args": [ + "foobar\n", + 15 + ], + "result": { + "index": 34, + "captureIndices": [ + { + "start": 15, + "end": 15, + "length": 0 + }, + { + "start": 15, + "end": 16, + "length": 1 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + }, + { + "start": 16, + "end": 20, + "length": 4 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + }, + { + "start": 20, + "end": 21, + "length": 1 + } + ] + } + }, + { + "args": [ + "foobar\n", + 75 + ], + "result": null + } + ] + }, + { "constractor": [ [ "(?!(<)\\s*(?:([_$A-Za-z][-_$0-9A-Za-z.]*)(?))", @@ -951,6 +1545,98 @@ ] ], "executions": [ + { + "args": [ + "foobar\n", + 15 + ], + "result": { + "index": 1, + "captureIndices": [ + { + "start": 15, + "end": 21, + "length": 6 + }, + { + "start": 15, + "end": 16, + "length": 1 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + }, + { + "start": 16, + "end": 20, + "length": 4 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + }, + { + "start": 20, + "end": 21, + "length": 1 + } + ] + } + }, + { + "args": [ + "foobar\n", + 75 + ], + "result": { + "index": 0, + "captureIndices": [ + { + "start": 75, + "end": 75, + "length": 0 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + } + ] + } + }, { "args": [ "foobar\n", @@ -1332,6 +2018,328 @@ ] } }, + { + "args": [ + "foobar\n", + 68 + ], + "result": { + "index": 0, + "captureIndices": [ + { + "start": 68, + "end": 75, + "length": 7 + }, + { + "start": 68, + "end": 70, + "length": 2 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + }, + { + "start": 70, + "end": 74, + "length": 4 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + }, + { + "start": 74, + "end": 75, + "length": 1 + } + ] + } + }, + { + "args": [ + "foobar\n", + 21 + ], + "result": { + "index": 1, + "captureIndices": [ + { + "start": 21, + "end": 27, + "length": 6 + }, + { + "start": 21, + "end": 22, + "length": 1 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + }, + { + "start": 22, + "end": 26, + "length": 4 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + }, + { + "start": 26, + "end": 27, + "length": 1 + } + ] + } + }, + { + "args": [ + "foobar\n", + 27 + ], + "result": { + "index": 1, + "captureIndices": [ + { + "start": 27, + "end": 34, + "length": 7 + }, + { + "start": 27, + "end": 28, + "length": 1 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + }, + { + "start": 28, + "end": 33, + "length": 5 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + }, + { + "start": 33, + "end": 34, + "length": 1 + } + ] + } + }, + { + "args": [ + "foobar\n", + 34 + ], + "result": { + "index": 0, + "captureIndices": [ + { + "start": 37, + "end": 45, + "length": 8 + }, + { + "start": 37, + "end": 39, + "length": 2 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + }, + { + "start": 39, + "end": 44, + "length": 5 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + }, + { + "start": 44, + "end": 45, + "length": 1 + } + ] + } + }, + { + "args": [ + "foobar\n", + 45 + ], + "result": { + "index": 0, + "captureIndices": [ + { + "start": 45, + "end": 52, + "length": 7 + }, + { + "start": 45, + "end": 47, + "length": 2 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + }, + { + "start": 47, + "end": 51, + "length": 4 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + }, + { + "start": 51, + "end": 52, + "length": 1 + } + ] + } + }, + { + "args": [ + "foobar\n", + 52 + ], + "result": { + "index": 1, + "captureIndices": [ + { + "start": 52, + "end": 58, + "length": 6 + }, + { + "start": 52, + "end": 53, + "length": 1 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + }, + { + "start": 53, + "end": 57, + "length": 4 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + }, + { + "start": 57, + "end": 58, + "length": 1 + } + ] + } + }, + { + "args": [ + "foobar\n", + 58 + ], + "result": { + "index": 0, + "captureIndices": [ + { + "start": 61, + "end": 68, + "length": 7 + }, + { + "start": 61, + "end": 63, + "length": 2 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + }, + { + "start": 63, + "end": 67, + "length": 4 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + }, + { + "start": 67, + "end": 68, + "length": 1 + } + ] + } + }, { "args": [ "foobar\n", diff --git a/packages/engine-javascript/test/__records__/json-basic.json b/packages/engine-javascript/test/__records__/json-basic.json index ffc1b211b..40180a2ce 100644 --- a/packages/engine-javascript/test/__records__/json-basic.json +++ b/packages/engine-javascript/test/__records__/json-basic.json @@ -36,18 +36,810 @@ ], "result": null }, + { + "args": [ + "{\"foo\":{\"bar\":1}}\n", + 0 + ], + "result": { + "index": 4, + "captureIndices": [ + { + "start": 0, + "end": 1, + "length": 1 + } + ] + } + }, + { + "args": [ + "{\"foo\":{\"bar\":1}}\n", + 17 + ], + "result": null + }, + { + "args": [ + "[undefined, null, true, false, 0, 1, 1.1, \"foo\", [], {}]\n", + 0 + ], + "result": { + "index": 3, + "captureIndices": [ + { + "start": 0, + "end": 1, + "length": 1 + } + ] + } + }, + { + "args": [ + "[undefined, null, true, false, 0, 1, 1.1, \"foo\", [], {}]\n", + 56 + ], + "result": null + }, + { + "args": [ + "[undefined, null, true, false, 0, 1, 1.1, \"foo\", [], {}]\n", + 0 + ], + "result": { + "index": 3, + "captureIndices": [ + { + "start": 0, + "end": 1, + "length": 1 + } + ] + } + }, + { + "args": [ + "[undefined, null, true, false, 0, 1, 1.1, \"foo\", [], {}]\n", + 56 + ], + "result": null + } + ] + }, + { + "constractor": [ + [ + "\\}", + "\"", + "/\\*\\*(?!/)", + "/\\*", + "(//).*$\\n?", + ":", + "[^\\s}]" + ] + ], + "executions": [ + { + "args": [ + "{\"foo\":{\"bar\":1}}\n", + 1 + ], + "result": { + "index": 1, + "captureIndices": [ + { + "start": 1, + "end": 2, + "length": 1 + } + ] + } + }, + { + "args": [ + "{\"foo\":{\"bar\":1}}\n", + 6 + ], + "result": { + "index": 5, + "captureIndices": [ + { + "start": 6, + "end": 7, + "length": 1 + } + ] + } + }, + { + "args": [ + "{\"foo\":{\"bar\":1}}\n", + 8 + ], + "result": { + "index": 1, + "captureIndices": [ + { + "start": 8, + "end": 9, + "length": 1 + } + ] + } + }, + { + "args": [ + "{\"foo\":{\"bar\":1}}\n", + 13 + ], + "result": { + "index": 5, + "captureIndices": [ + { + "start": 13, + "end": 14, + "length": 1 + } + ] + } + }, + { + "args": [ + "{\"foo\":{\"bar\":1}}\n", + 15 + ], + "result": { + "index": 0, + "captureIndices": [ + { + "start": 15, + "end": 16, + "length": 1 + } + ] + } + }, + { + "args": [ + "{\"foo\":{\"bar\":1}}\n", + 16 + ], + "result": { + "index": 0, + "captureIndices": [ + { + "start": 16, + "end": 17, + "length": 1 + } + ] + } + }, + { + "args": [ + "{\"foo\":{\"bar\":1}}\n", + 1 + ], + "result": { + "index": 1, + "captureIndices": [ + { + "start": 1, + "end": 2, + "length": 1 + } + ] + } + }, + { + "args": [ + "{\"foo\":{\"bar\":1}}\n", + 6 + ], + "result": { + "index": 5, + "captureIndices": [ + { + "start": 6, + "end": 7, + "length": 1 + } + ] + } + }, + { + "args": [ + "{\"foo\":{\"bar\":1}}\n", + 8 + ], + "result": { + "index": 1, + "captureIndices": [ + { + "start": 8, + "end": 9, + "length": 1 + } + ] + } + }, + { + "args": [ + "{\"foo\":{\"bar\":1}}\n", + 13 + ], + "result": { + "index": 5, + "captureIndices": [ + { + "start": 13, + "end": 14, + "length": 1 + } + ] + } + }, + { + "args": [ + "{\"foo\":{\"bar\":1}}\n", + 15 + ], + "result": { + "index": 0, + "captureIndices": [ + { + "start": 15, + "end": 16, + "length": 1 + } + ] + } + }, + { + "args": [ + "{\"foo\":{\"bar\":1}}\n", + 16 + ], + "result": { + "index": 0, + "captureIndices": [ + { + "start": 16, + "end": 17, + "length": 1 + } + ] + } + }, + { + "args": [ + "[undefined, null, true, false, 0, 1, 1.1, \"foo\", [], {}]\n", + 54 + ], + "result": { + "index": 0, + "captureIndices": [ + { + "start": 54, + "end": 55, + "length": 1 + } + ] + } + }, + { + "args": [ + "[undefined, null, true, false, 0, 1, 1.1, \"foo\", [], {}]\n", + 54 + ], + "result": { + "index": 0, + "captureIndices": [ + { + "start": 54, + "end": 55, + "length": 1 + } + ] + } + } + ] + }, + { + "constractor": [ + [ + "\"", + "\\\\(?:[\"\\\\/bfnrt]|u[0-9a-fA-F]{4})", + "\\\\." + ] + ], + "executions": [ + { + "args": [ + "{\"foo\":{\"bar\":1}}\n", + 2 + ], + "result": { + "index": 0, + "captureIndices": [ + { + "start": 5, + "end": 6, + "length": 1 + } + ] + } + }, + { + "args": [ + "{\"foo\":{\"bar\":1}}\n", + 9 + ], + "result": { + "index": 0, + "captureIndices": [ + { + "start": 12, + "end": 13, + "length": 1 + } + ] + } + }, + { + "args": [ + "{\"foo\":{\"bar\":1}}\n", + 2 + ], + "result": { + "index": 0, + "captureIndices": [ + { + "start": 5, + "end": 6, + "length": 1 + } + ] + } + }, + { + "args": [ + "{\"foo\":{\"bar\":1}}\n", + 9 + ], + "result": { + "index": 0, + "captureIndices": [ + { + "start": 12, + "end": 13, + "length": 1 + } + ] + } + } + ] + }, + { + "constractor": [ + [ + "(,)|(?=\\})", + "\\b(?:true|false|null)\\b", + "-?(?:0|[1-9]\\d*)(?:(?:\\.\\d+)?(?:[eE][+-]?\\d+)?)?", + "\"", + "\\[", + "\\{", + "/\\*\\*(?!/)", + "/\\*", + "(//).*$\\n?", + "[^\\s,]" + ] + ], + "executions": [ + { + "args": [ + "{\"foo\":{\"bar\":1}}\n", + 7 + ], + "result": { + "index": 5, + "captureIndices": [ + { + "start": 7, + "end": 8, + "length": 1 + } + ] + } + }, + { + "args": [ + "{\"foo\":{\"bar\":1}}\n", + 14 + ], + "result": { + "index": 2, + "captureIndices": [ + { + "start": 14, + "end": 15, + "length": 1 + } + ] + } + }, + { + "args": [ + "{\"foo\":{\"bar\":1}}\n", + 15 + ], + "result": { + "index": 0, + "captureIndices": [ + { + "start": 15, + "end": 15, + "length": 0 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + } + ] + } + }, + { + "args": [ + "{\"foo\":{\"bar\":1}}\n", + 16 + ], + "result": { + "index": 0, + "captureIndices": [ + { + "start": 16, + "end": 16, + "length": 0 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + } + ] + } + }, + { + "args": [ + "{\"foo\":{\"bar\":1}}\n", + 7 + ], + "result": { + "index": 5, + "captureIndices": [ + { + "start": 7, + "end": 8, + "length": 1 + } + ] + } + }, + { + "args": [ + "{\"foo\":{\"bar\":1}}\n", + 14 + ], + "result": { + "index": 2, + "captureIndices": [ + { + "start": 14, + "end": 15, + "length": 1 + } + ] + } + }, + { + "args": [ + "{\"foo\":{\"bar\":1}}\n", + 15 + ], + "result": { + "index": 0, + "captureIndices": [ + { + "start": 15, + "end": 15, + "length": 0 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + } + ] + } + }, + { + "args": [ + "{\"foo\":{\"bar\":1}}\n", + 16 + ], + "result": { + "index": 0, + "captureIndices": [ + { + "start": 16, + "end": 16, + "length": 0 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + } + ] + } + } + ] + }, + { + "constractor": [ + [ + "\\]", + "\\b(?:true|false|null)\\b", + "-?(?:0|[1-9]\\d*)(?:(?:\\.\\d+)?(?:[eE][+-]?\\d+)?)?", + "\"", + "\\[", + "\\{", + "/\\*\\*(?!/)", + "/\\*", + "(//).*$\\n?", + ",", + "[^\\s\\]]" + ] + ], + "executions": [ + { + "args": [ + "[undefined, null, true, false, 0, 1, 1.1, \"foo\", [], {}]\n", + 1 + ], + "result": { + "index": 10, + "captureIndices": [ + { + "start": 1, + "end": 2, + "length": 1 + } + ] + } + }, + { + "args": [ + "[undefined, null, true, false, 0, 1, 1.1, \"foo\", [], {}]\n", + 2 + ], + "result": { + "index": 10, + "captureIndices": [ + { + "start": 2, + "end": 3, + "length": 1 + } + ] + } + }, + { + "args": [ + "[undefined, null, true, false, 0, 1, 1.1, \"foo\", [], {}]\n", + 3 + ], + "result": { + "index": 10, + "captureIndices": [ + { + "start": 3, + "end": 4, + "length": 1 + } + ] + } + }, + { + "args": [ + "[undefined, null, true, false, 0, 1, 1.1, \"foo\", [], {}]\n", + 4 + ], + "result": { + "index": 10, + "captureIndices": [ + { + "start": 4, + "end": 5, + "length": 1 + } + ] + } + }, + { + "args": [ + "[undefined, null, true, false, 0, 1, 1.1, \"foo\", [], {}]\n", + 5 + ], + "result": { + "index": 10, + "captureIndices": [ + { + "start": 5, + "end": 6, + "length": 1 + } + ] + } + }, + { + "args": [ + "[undefined, null, true, false, 0, 1, 1.1, \"foo\", [], {}]\n", + 6 + ], + "result": { + "index": 10, + "captureIndices": [ + { + "start": 6, + "end": 7, + "length": 1 + } + ] + } + }, + { + "args": [ + "[undefined, null, true, false, 0, 1, 1.1, \"foo\", [], {}]\n", + 7 + ], + "result": { + "index": 10, + "captureIndices": [ + { + "start": 7, + "end": 8, + "length": 1 + } + ] + } + }, + { + "args": [ + "[undefined, null, true, false, 0, 1, 1.1, \"foo\", [], {}]\n", + 8 + ], + "result": { + "index": 10, + "captureIndices": [ + { + "start": 8, + "end": 9, + "length": 1 + } + ] + } + }, + { + "args": [ + "[undefined, null, true, false, 0, 1, 1.1, \"foo\", [], {}]\n", + 9 + ], + "result": { + "index": 10, + "captureIndices": [ + { + "start": 9, + "end": 10, + "length": 1 + } + ] + } + }, + { + "args": [ + "[undefined, null, true, false, 0, 1, 1.1, \"foo\", [], {}]\n", + 10 + ], + "result": { + "index": 9, + "captureIndices": [ + { + "start": 10, + "end": 11, + "length": 1 + } + ] + } + }, + { + "args": [ + "[undefined, null, true, false, 0, 1, 1.1, \"foo\", [], {}]\n", + 11 + ], + "result": { + "index": 1, + "captureIndices": [ + { + "start": 12, + "end": 16, + "length": 4 + } + ] + } + }, + { + "args": [ + "[undefined, null, true, false, 0, 1, 1.1, \"foo\", [], {}]\n", + 16 + ], + "result": { + "index": 9, + "captureIndices": [ + { + "start": 16, + "end": 17, + "length": 1 + } + ] + } + }, + { + "args": [ + "[undefined, null, true, false, 0, 1, 1.1, \"foo\", [], {}]\n", + 17 + ], + "result": { + "index": 1, + "captureIndices": [ + { + "start": 18, + "end": 22, + "length": 4 + } + ] + } + }, + { + "args": [ + "[undefined, null, true, false, 0, 1, 1.1, \"foo\", [], {}]\n", + 22 + ], + "result": { + "index": 9, + "captureIndices": [ + { + "start": 22, + "end": 23, + "length": 1 + } + ] + } + }, { "args": [ "[undefined, null, true, false, 0, 1, 1.1, \"foo\", [], {}]\n", - 0 + 23 ], "result": { - "index": 3, + "index": 1, "captureIndices": [ { - "start": 0, - "end": 1, - "length": 1 + "start": 24, + "end": 29, + "length": 5 } ] } @@ -55,36 +847,30 @@ { "args": [ "[undefined, null, true, false, 0, 1, 1.1, \"foo\", [], {}]\n", - 56 + 29 ], - "result": null - } - ] - }, - { - "constractor": [ - [ - "\\}", - "\"", - "/\\*\\*(?!/)", - "/\\*", - "(//).*$\\n?", - ":", - "[^\\s}]" - ] - ], - "executions": [ + "result": { + "index": 9, + "captureIndices": [ + { + "start": 29, + "end": 30, + "length": 1 + } + ] + } + }, { "args": [ - "{\"foo\":{\"bar\":1}}\n", - 1 + "[undefined, null, true, false, 0, 1, 1.1, \"foo\", [], {}]\n", + 30 ], "result": { - "index": 1, + "index": 2, "captureIndices": [ { - "start": 1, - "end": 2, + "start": 31, + "end": 32, "length": 1 } ] @@ -92,15 +878,15 @@ }, { "args": [ - "{\"foo\":{\"bar\":1}}\n", - 6 + "[undefined, null, true, false, 0, 1, 1.1, \"foo\", [], {}]\n", + 32 ], "result": { - "index": 5, + "index": 9, "captureIndices": [ { - "start": 6, - "end": 7, + "start": 32, + "end": 33, "length": 1 } ] @@ -108,15 +894,15 @@ }, { "args": [ - "{\"foo\":{\"bar\":1}}\n", - 8 + "[undefined, null, true, false, 0, 1, 1.1, \"foo\", [], {}]\n", + 33 ], "result": { - "index": 1, + "index": 2, "captureIndices": [ { - "start": 8, - "end": 9, + "start": 34, + "end": 35, "length": 1 } ] @@ -124,15 +910,15 @@ }, { "args": [ - "{\"foo\":{\"bar\":1}}\n", - 13 + "[undefined, null, true, false, 0, 1, 1.1, \"foo\", [], {}]\n", + 35 ], "result": { - "index": 5, + "index": 9, "captureIndices": [ { - "start": 13, - "end": 14, + "start": 35, + "end": 36, "length": 1 } ] @@ -140,31 +926,31 @@ }, { "args": [ - "{\"foo\":{\"bar\":1}}\n", - 15 + "[undefined, null, true, false, 0, 1, 1.1, \"foo\", [], {}]\n", + 36 ], "result": { - "index": 0, + "index": 2, "captureIndices": [ { - "start": 15, - "end": 16, - "length": 1 + "start": 37, + "end": 40, + "length": 3 } ] } }, { "args": [ - "{\"foo\":{\"bar\":1}}\n", - 16 + "[undefined, null, true, false, 0, 1, 1.1, \"foo\", [], {}]\n", + 40 ], "result": { - "index": 0, + "index": 9, "captureIndices": [ { - "start": 16, - "end": 17, + "start": 40, + "end": 41, "length": 1 } ] @@ -173,41 +959,30 @@ { "args": [ "[undefined, null, true, false, 0, 1, 1.1, \"foo\", [], {}]\n", - 54 + 41 ], "result": { - "index": 0, + "index": 3, "captureIndices": [ { - "start": 54, - "end": 55, + "start": 42, + "end": 43, "length": 1 } ] } - } - ] - }, - { - "constractor": [ - [ - "\"", - "\\\\(?:[\"\\\\/bfnrt]|u[0-9a-fA-F]{4})", - "\\\\." - ] - ], - "executions": [ + }, { "args": [ - "{\"foo\":{\"bar\":1}}\n", - 2 + "[undefined, null, true, false, 0, 1, 1.1, \"foo\", [], {}]\n", + 47 ], "result": { - "index": 0, + "index": 9, "captureIndices": [ { - "start": 5, - "end": 6, + "start": 47, + "end": 48, "length": 1 } ] @@ -215,49 +990,31 @@ }, { "args": [ - "{\"foo\":{\"bar\":1}}\n", - 9 + "[undefined, null, true, false, 0, 1, 1.1, \"foo\", [], {}]\n", + 48 ], "result": { - "index": 0, + "index": 4, "captureIndices": [ { - "start": 12, - "end": 13, + "start": 49, + "end": 50, "length": 1 } ] } - } - ] - }, - { - "constractor": [ - [ - "(,)|(?=\\})", - "\\b(?:true|false|null)\\b", - "-?(?:0|[1-9]\\d*)(?:(?:\\.\\d+)?(?:[eE][+-]?\\d+)?)?", - "\"", - "\\[", - "\\{", - "/\\*\\*(?!/)", - "/\\*", - "(//).*$\\n?", - "[^\\s,]" - ] - ], - "executions": [ + }, { "args": [ - "{\"foo\":{\"bar\":1}}\n", - 7 + "[undefined, null, true, false, 0, 1, 1.1, \"foo\", [], {}]\n", + 50 ], "result": { - "index": 5, + "index": 0, "captureIndices": [ { - "start": 7, - "end": 8, + "start": 50, + "end": 51, "length": 1 } ] @@ -265,15 +1022,15 @@ }, { "args": [ - "{\"foo\":{\"bar\":1}}\n", - 14 + "[undefined, null, true, false, 0, 1, 1.1, \"foo\", [], {}]\n", + 51 ], "result": { - "index": 2, + "index": 9, "captureIndices": [ { - "start": 14, - "end": 15, + "start": 51, + "end": 52, "length": 1 } ] @@ -281,65 +1038,36 @@ }, { "args": [ - "{\"foo\":{\"bar\":1}}\n", - 15 + "[undefined, null, true, false, 0, 1, 1.1, \"foo\", [], {}]\n", + 52 ], "result": { - "index": 0, + "index": 5, "captureIndices": [ { - "start": 15, - "end": 15, - "length": 0 - }, - { - "start": 4294967295, - "end": 4294967295, - "length": 0 + "start": 53, + "end": 54, + "length": 1 } ] } }, { "args": [ - "{\"foo\":{\"bar\":1}}\n", - 16 + "[undefined, null, true, false, 0, 1, 1.1, \"foo\", [], {}]\n", + 55 ], "result": { "index": 0, "captureIndices": [ { - "start": 16, - "end": 16, - "length": 0 - }, - { - "start": 4294967295, - "end": 4294967295, - "length": 0 + "start": 55, + "end": 56, + "length": 1 } ] } - } - ] - }, - { - "constractor": [ - [ - "\\]", - "\\b(?:true|false|null)\\b", - "-?(?:0|[1-9]\\d*)(?:(?:\\.\\d+)?(?:[eE][+-]?\\d+)?)?", - "\"", - "\\[", - "\\{", - "/\\*\\*(?!/)", - "/\\*", - "(//).*$\\n?", - ",", - "[^\\s\\]]" - ] - ], - "executions": [ + }, { "args": [ "[undefined, null, true, false, 0, 1, 1.1, \"foo\", [], {}]\n", @@ -815,6 +1543,22 @@ ] ], "executions": [ + { + "args": [ + "[undefined, null, true, false, 0, 1, 1.1, \"foo\", [], {}]\n", + 43 + ], + "result": { + "index": 0, + "captureIndices": [ + { + "start": 46, + "end": 47, + "length": 1 + } + ] + } + }, { "args": [ "[undefined, null, true, false, 0, 1, 1.1, \"foo\", [], {}]\n", diff --git a/packages/engine-javascript/test/__records__/jsonc.json b/packages/engine-javascript/test/__records__/jsonc.json index 16dd07b8c..1be3268de 100644 --- a/packages/engine-javascript/test/__records__/jsonc.json +++ b/packages/engine-javascript/test/__records__/jsonc.json @@ -13,6 +13,57 @@ ] ], "executions": [ + { + "args": [ + "// comment\n", + 0 + ], + "result": { + "index": 7, + "captureIndices": [ + { + "start": 0, + "end": 11, + "length": 11 + }, + { + "start": 0, + "end": 2, + "length": 2 + } + ] + } + }, + { + "args": [ + "// comment\n", + 11 + ], + "result": null + }, + { + "args": [ + "{\"foo\":\"bar\"}\n", + 0 + ], + "result": { + "index": 4, + "captureIndices": [ + { + "start": 0, + "end": 1, + "length": 1 + } + ] + } + }, + { + "args": [ + "{\"foo\":\"bar\"}\n", + 13 + ], + "result": null + }, { "args": [ "// comment\n", @@ -79,6 +130,54 @@ ] ], "executions": [ + { + "args": [ + "{\"foo\":\"bar\"}\n", + 1 + ], + "result": { + "index": 1, + "captureIndices": [ + { + "start": 1, + "end": 2, + "length": 1 + } + ] + } + }, + { + "args": [ + "{\"foo\":\"bar\"}\n", + 6 + ], + "result": { + "index": 5, + "captureIndices": [ + { + "start": 6, + "end": 7, + "length": 1 + } + ] + } + }, + { + "args": [ + "{\"foo\":\"bar\"}\n", + 12 + ], + "result": { + "index": 0, + "captureIndices": [ + { + "start": 12, + "end": 13, + "length": 1 + } + ] + } + }, { "args": [ "{\"foo\":\"bar\"}\n", @@ -138,6 +237,22 @@ ] ], "executions": [ + { + "args": [ + "{\"foo\":\"bar\"}\n", + 2 + ], + "result": { + "index": 0, + "captureIndices": [ + { + "start": 5, + "end": 6, + "length": 1 + } + ] + } + }, { "args": [ "{\"foo\":\"bar\"}\n", @@ -172,6 +287,43 @@ ] ], "executions": [ + { + "args": [ + "{\"foo\":\"bar\"}\n", + 7 + ], + "result": { + "index": 3, + "captureIndices": [ + { + "start": 7, + "end": 8, + "length": 1 + } + ] + } + }, + { + "args": [ + "{\"foo\":\"bar\"}\n", + 12 + ], + "result": { + "index": 0, + "captureIndices": [ + { + "start": 12, + "end": 12, + "length": 0 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + } + ] + } + }, { "args": [ "{\"foo\":\"bar\"}\n", @@ -220,6 +372,22 @@ ] ], "executions": [ + { + "args": [ + "{\"foo\":\"bar\"}\n", + 8 + ], + "result": { + "index": 0, + "captureIndices": [ + { + "start": 11, + "end": 12, + "length": 1 + } + ] + } + }, { "args": [ "{\"foo\":\"bar\"}\n", diff --git a/packages/engine-javascript/test/__records__/sql.json b/packages/engine-javascript/test/__records__/sql.json index 6adf528c3..f6676ed3f 100644 --- a/packages/engine-javascript/test/__records__/sql.json +++ b/packages/engine-javascript/test/__records__/sql.json @@ -238,6 +238,513 @@ ], "result": null }, + { + "args": [ + "SELECT * FROM foo\n", + 0 + ], + "result": { + "index": 12, + "captureIndices": [ + { + "start": 0, + "end": 6, + "length": 6 + }, + { + "start": 0, + "end": 6, + "length": 6 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + } + ] + } + }, + { + "args": [ + "SELECT * FROM foo\n", + 6 + ], + "result": { + "index": 21, + "captureIndices": [ + { + "start": 7, + "end": 8, + "length": 1 + } + ] + } + }, + { + "args": [ + "SELECT * FROM foo\n", + 8 + ], + "result": { + "index": 12, + "captureIndices": [ + { + "start": 9, + "end": 13, + "length": 4 + }, + { + "start": 9, + "end": 13, + "length": 4 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + } + ] + } + }, + { + "args": [ + "SELECT * FROM foo\n", + 13 + ], + "result": null + }, + { + "args": [ + "USE AdventureWorks2022;\n", + 0 + ], + "result": { + "index": 56, + "captureIndices": [ + { + "start": 0, + "end": 3, + "length": 3 + }, + { + "start": 0, + "end": 3, + "length": 3 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + } + ] + } + }, + { + "args": [ + "USE AdventureWorks2022;\n", + 3 + ], + "result": null + }, + { + "args": [ + "GO\n", + 0 + ], + "result": { + "index": 56, + "captureIndices": [ + { + "start": 0, + "end": 2, + "length": 2 + }, + { + "start": 0, + "end": 2, + "length": 2 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + } + ] + } + }, + { + "args": [ + "GO\n", + 2 + ], + "result": null + }, + { + "args": [ + "IF OBJECT_ID('dbo.NewProducts', 'U') IS NOT NULL\n", + 0 + ], + "result": { + "index": 56, + "captureIndices": [ + { + "start": 0, + "end": 2, + "length": 2 + }, + { + "start": 0, + "end": 2, + "length": 2 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + } + ] + } + }, + { + "args": [ + "IF OBJECT_ID('dbo.NewProducts', 'U') IS NOT NULL\n", + 2 + ], + "result": { + "index": 39, + "captureIndices": [ + { + "start": 3, + "end": 13, + "length": 10 + }, + { + "start": 3, + "end": 12, + "length": 9 + } + ] + } + }, + { + "args": [ + "IF OBJECT_ID('dbo.NewProducts', 'U') IS NOT NULL\n", + 13 + ], + "result": { + "index": 47, + "captureIndices": [ + { + "start": 13, + "end": 30, + "length": 17 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + }, + { + "start": 13, + "end": 14, + "length": 1 + }, + { + "start": 29, + "end": 30, + "length": 1 + } + ] + } + }, + { + "args": [ + "IF OBJECT_ID('dbo.NewProducts', 'U') IS NOT NULL\n", + 30 + ], + "result": { + "index": 47, + "captureIndices": [ + { + "start": 32, + "end": 35, + "length": 3 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + }, + { + "start": 32, + "end": 33, + "length": 1 + }, + { + "start": 34, + "end": 35, + "length": 1 + } + ] + } + }, + { + "args": [ + "IF OBJECT_ID('dbo.NewProducts', 'U') IS NOT NULL\n", + 35 + ], + "result": { + "index": 13, + "captureIndices": [ + { + "start": 37, + "end": 48, + "length": 11 + }, + { + "start": 37, + "end": 48, + "length": 11 + }, + { + "start": 37, + "end": 44, + "length": 7 + }, + { + "start": 37, + "end": 40, + "length": 3 + } + ] + } + }, + { + "args": [ + "IF OBJECT_ID('dbo.NewProducts', 'U') IS NOT NULL\n", + 48 + ], + "result": null + }, + { + "args": [ + "DROP TABLE dbo.NewProducts;\n", + 0 + ], + "result": { + "index": 6, + "captureIndices": [ + { + "start": 0, + "end": 10, + "length": 10 + }, + { + "start": 0, + "end": 4, + "length": 4 + }, + { + "start": 5, + "end": 10, + "length": 5 + } + ] + } + }, + { + "args": [ + "DROP TABLE dbo.NewProducts;\n", + 10 + ], + "result": { + "index": 46, + "captureIndices": [ + { + "start": 11, + "end": 26, + "length": 15 + }, + { + "start": 11, + "end": 14, + "length": 3 + }, + { + "start": 15, + "end": 26, + "length": 11 + } + ] + } + }, + { + "args": [ + "DROP TABLE dbo.NewProducts;\n", + 26 + ], + "result": null + }, { "args": [ "USE AdventureWorks2022;\n", diff --git a/packages/engine-javascript/test/__records__/toml.json b/packages/engine-javascript/test/__records__/toml.json index 7ef0f1501..256ee14ba 100644 --- a/packages/engine-javascript/test/__records__/toml.json +++ b/packages/engine-javascript/test/__records__/toml.json @@ -138,6 +138,138 @@ ] } }, + { + "args": [ + "name = \"Tom Preston-Werner\"\n", + 27 + ], + "result": null + }, + { + "args": [ + "# This is a TOML document\n", + 0 + ], + "result": { + "index": 0, + "captureIndices": [ + { + "start": 0, + "end": 0, + "length": 0 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + } + ] + } + }, + { + "args": [ + "# This is a TOML document\n", + 26 + ], + "result": null + }, + { + "args": [ + "title = \"TOML Example\"\n", + 0 + ], + "result": { + "index": 3, + "captureIndices": [ + { + "start": 0, + "end": 8, + "length": 8 + }, + { + "start": 0, + "end": 5, + "length": 5 + }, + { + "start": 6, + "end": 7, + "length": 1 + } + ] + } + }, + { + "args": [ + "title = \"TOML Example\"\n", + 22 + ], + "result": null + }, + { + "args": [ + "[owner]\n", + 0 + ], + "result": { + "index": 1, + "captureIndices": [ + { + "start": 0, + "end": 7, + "length": 7 + }, + { + "start": 0, + "end": 1, + "length": 1 + }, + { + "start": 1, + "end": 6, + "length": 5 + }, + { + "start": 6, + "end": 7, + "length": 1 + } + ] + } + }, + { + "args": [ + "[owner]\n", + 7 + ], + "result": null + }, + { + "args": [ + "name = \"Tom Preston-Werner\"\n", + 0 + ], + "result": { + "index": 3, + "captureIndices": [ + { + "start": 0, + "end": 7, + "length": 7 + }, + { + "start": 0, + "end": 4, + "length": 4 + }, + { + "start": 5, + "end": 6, + "length": 1 + } + ] + } + }, { "args": [ "name = \"Tom Preston-Werner\"\n", @@ -155,6 +287,22 @@ ] ], "executions": [ + { + "args": [ + "# This is a TOML document\n", + 0 + ], + "result": { + "index": 1, + "captureIndices": [ + { + "start": 0, + "end": 1, + "length": 1 + } + ] + } + }, { "args": [ "# This is a TOML document\n", @@ -180,6 +328,22 @@ ] ], "executions": [ + { + "args": [ + "# This is a TOML document\n", + 1 + ], + "result": { + "index": 0, + "captureIndices": [ + { + "start": 25, + "end": 26, + "length": 1 + } + ] + } + }, { "args": [ "# This is a TOML document\n", @@ -206,6 +370,22 @@ ] ], "executions": [ + { + "args": [ + "# This is a TOML document\n", + 26 + ], + "result": { + "index": 0, + "captureIndices": [ + { + "start": 26, + "end": 26, + "length": 0 + } + ] + } + }, { "args": [ "# This is a TOML document\n", @@ -246,6 +426,38 @@ ] ], "executions": [ + { + "args": [ + "title = \"TOML Example\"\n", + 8 + ], + "result": { + "index": 2, + "captureIndices": [ + { + "start": 8, + "end": 9, + "length": 1 + } + ] + } + }, + { + "args": [ + "name = \"Tom Preston-Werner\"\n", + 7 + ], + "result": { + "index": 2, + "captureIndices": [ + { + "start": 7, + "end": 8, + "length": 1 + } + ] + } + }, { "args": [ "title = \"TOML Example\"\n", @@ -289,6 +501,38 @@ ] ], "executions": [ + { + "args": [ + "title = \"TOML Example\"\n", + 9 + ], + "result": { + "index": 0, + "captureIndices": [ + { + "start": 21, + "end": 22, + "length": 1 + } + ] + } + }, + { + "args": [ + "name = \"Tom Preston-Werner\"\n", + 8 + ], + "result": { + "index": 0, + "captureIndices": [ + { + "start": 26, + "end": 27, + "length": 1 + } + ] + } + }, { "args": [ "title = \"TOML Example\"\n", @@ -345,6 +589,38 @@ ] ], "executions": [ + { + "args": [ + "title = \"TOML Example\"\n", + 22 + ], + "result": { + "index": 0, + "captureIndices": [ + { + "start": 22, + "end": 22, + "length": 0 + } + ] + } + }, + { + "args": [ + "name = \"Tom Preston-Werner\"\n", + 27 + ], + "result": { + "index": 0, + "captureIndices": [ + { + "start": 27, + "end": 27, + "length": 0 + } + ] + } + }, { "args": [ "title = \"TOML Example\"\n", @@ -386,6 +662,29 @@ ] ], "executions": [ + { + "args": [ + "[owner", + 1 + ], + "result": { + "index": 0, + "captureIndices": [ + { + "start": 1, + "end": 6, + "length": 5 + } + ] + } + }, + { + "args": [ + "[owner", + 6 + ], + "result": null + }, { "args": [ "[owner", diff --git a/packages/engine-javascript/test/__records__/ts-basic.json b/packages/engine-javascript/test/__records__/ts-basic.json index fcd91a3f0..bdce486dc 100644 --- a/packages/engine-javascript/test/__records__/ts-basic.json +++ b/packages/engine-javascript/test/__records__/ts-basic.json @@ -133,6 +133,37 @@ ] ], "executions": [ + { + "args": [ + "const foo: string = \"bar\"\n", + 0 + ], + "result": { + "index": 3, + "captureIndices": [ + { + "start": 0, + "end": 0, + "length": 0 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + }, + { + "start": 0, + "end": 5, + "length": 5 + } + ] + } + }, { "args": [ "const foo: string = \"bar\"\n", @@ -258,6 +289,130 @@ ] } }, + { + "args": [ + "const foo: string = \"bar\"\n", + 25 + ], + "result": { + "index": 0, + "captureIndices": [ + { + "start": 25, + "end": 25, + "length": 0 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + }, + { + "start": 25, + "end": 25, + "length": 0 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + }, + { + "start": 25, + "end": 25, + "length": 0 + } + ] + } + }, + { + "args": [ + "const foo: string = \"bar\"\n", + 0 + ], + "result": { + "index": 1, + "captureIndices": [ + { + "start": 0, + "end": 6, + "length": 6 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + }, + { + "start": 0, + "end": 5, + "length": 5 + } + ] + } + }, + { + "args": [ + "const foo: string = \"bar\"\n", + 6 + ], + "result": { + "index": 5, + "captureIndices": [ + { + "start": 6, + "end": 9, + "length": 3 + }, + { + "start": 6, + "end": 9, + "length": 3 + } + ] + } + }, + { + "args": [ + "const foo: string = \"bar\"\n", + 18 + ], + "result": { + "index": 6, + "captureIndices": [ + { + "start": 18, + "end": 19, + "length": 1 + }, + { + "start": 18, + "end": 19, + "length": 1 + } + ] + } + }, { "args": [ "const foo: string = \"bar\"\n", @@ -318,6 +473,22 @@ ] ], "executions": [ + { + "args": [ + "const foo: string = \"bar\"\n", + 6 + ], + "result": { + "index": 0, + "captureIndices": [ + { + "start": 6, + "end": 6, + "length": 0 + } + ] + } + }, { "args": [ "const foo: string = \"bar\"\n", @@ -353,6 +524,58 @@ ] ], "executions": [ + { + "args": [ + "const foo: string = \"bar\"\n", + 9 + ], + "result": { + "index": 1, + "captureIndices": [ + { + "start": 9, + "end": 10, + "length": 1 + }, + { + "start": 9, + "end": 10, + "length": 1 + } + ] + } + }, + { + "args": [ + "const foo: string = \"bar\"\n", + 18 + ], + "result": { + "index": 0, + "captureIndices": [ + { + "start": 18, + "end": 18, + "length": 0 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + } + ] + } + }, { "args": [ "const foo: string = \"bar\"\n", @@ -450,6 +673,53 @@ ] ], "executions": [ + { + "args": [ + "const foo: string = \"bar\"\n", + 10 + ], + "result": { + "index": 13, + "captureIndices": [ + { + "start": 11, + "end": 17, + "length": 6 + }, + { + "start": 11, + "end": 17, + "length": 6 + } + ] + } + }, + { + "args": [ + "const foo: string = \"bar\"\n", + 17 + ], + "result": { + "index": 0, + "captureIndices": [ + { + "start": 18, + "end": 18, + "length": 0 + }, + { + "start": 18, + "end": 18, + "length": 0 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + } + ] + } + }, { "args": [ "const foo: string = \"bar\"\n", @@ -590,6 +860,48 @@ ] ], "executions": [ + { + "args": [ + "const foo: string = \"bar\"\n", + 19 + ], + "result": { + "index": 2, + "captureIndices": [ + { + "start": 20, + "end": 21, + "length": 1 + } + ] + } + }, + { + "args": [ + "const foo: string = \"bar\"\n", + 25 + ], + "result": { + "index": 0, + "captureIndices": [ + { + "start": 25, + "end": 25, + "length": 0 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + } + ] + } + }, { "args": [ "const foo: string = \"bar\"\n", @@ -642,6 +954,32 @@ ] ], "executions": [ + { + "args": [ + "const foo: string = \"bar\"\n", + 21 + ], + "result": { + "index": 0, + "captureIndices": [ + { + "start": 24, + "end": 25, + "length": 1 + }, + { + "start": 24, + "end": 25, + "length": 1 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + } + ] + } + }, { "args": [ "const foo: string = \"bar\"\n", @@ -804,6 +1142,13 @@ ] ], "executions": [ + { + "args": [ + "const foo: string = \"bar\"\n", + 25 + ], + "result": null + }, { "args": [ "const foo: string = \"bar\"\n", diff --git a/packages/engine-javascript/test/__records__/vue.json b/packages/engine-javascript/test/__records__/vue.json index 7b316f1dd..db8370822 100644 --- a/packages/engine-javascript/test/__records__/vue.json +++ b/packages/engine-javascript/test/__records__/vue.json @@ -200,6 +200,118 @@ ] } }, + { + "args": [ + "\n", + 9 + ], + "result": { + "index": 0, + "captureIndices": [ + { + "start": 9, + "end": 9, + "length": 0 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + } + ] + } + }, + { + "args": [ + "\n", @@ -506,36 +730,196 @@ } ] } - } - ] - }, - { - "constractor": [ - [ - "(?=[/]?>)", - "/\\*\\*(?!/)", - "(/\\*)(?:\\s*((@)internal)(?=\\s|(\\*/)))?", - "(^[ \\t]+)?((//)(?:\\s*((@)internal)(?=\\s|$))?)", - "<", - "\\s+" - ] - ], - "executions": [ + }, { "args": [ "\n", + 0 + ], + "result": { + "index": 0, + "captureIndices": [ + { + "start": 0, + "end": 9, + "length": 9 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + }, + { + "start": 0, + "end": 2, + "length": 2 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + }, + { + "start": 2, + "end": 8, + "length": 6 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + }, + { + "start": 8, + "end": 9, + "length": 1 + } + ] + } + } + ] + }, + { + "constractor": [ + [ + "(?=[/]?>)", + "/\\*\\*(?!/)", + "(/\\*)(?:\\s*((@)internal)(?=\\s|(\\*/)))?", + "(^[ \\t]+)?((//)(?:\\s*((@)internal)(?=\\s|$))?)", + "<", + "\\s+" + ] + ], + "executions": [ + { + "args": [ + "\n", + 0 + ], + "result": { + "index": 0, + "captureIndices": [ + { + "start": 0, + "end": 0, + "length": 0 + } + ] + } + }, { "args": [ "\n", + 9 + ], + "result": null + }, + { + "args": [ + "\n", + 11 + ], + "result": null + }, + { + "args": [ + "\n", + 11 + ], + "result": null + } + ] + }, + { + "constractor": [ + [ + "(?!(<)\\s*(?:([_$A-Za-z][-_$0-9A-Za-z.]*)(?))", + "(<)\\s*(?:([_$A-Za-z][-_$0-9A-Za-z.]*)(?)" + ] + ], + "executions": [ + { + "args": [ + "\n", + 11 + ], + "result": { + "index": 0, + "captureIndices": [ + { + "start": 11, + "end": 11, + "length": 0 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + } + ] + } + }, { "args": [ "\n", + 0 + ], + "result": { + "index": 0, + "captureIndices": [ + { + "start": 0, + "end": 11, + "length": 11 + }, + { + "start": 0, + "end": 2, + "length": 2 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + }, + { + "start": 2, + "end": 10, + "length": 8 + }, + { + "start": 4294967295, + "end": 4294967295, + "length": 0 + }, + { + "start": 10, + "end": 11, + "length": 1 + } + ] + } + }, + { + "args": [ + "\n", @@ -1297,6 +2112,43 @@ ] ], "executions": [ + { + "args": [ + "
{{ foo }}
\n", + 7 + ], + "result": { + "index": 13, + "captureIndices": [ + { + "start": 8, + "end": 12, + "length": 4 + }, + { + "start": 8, + "end": 11, + "length": 3 + } + ] + } + }, + { + "args": [ + "
{{ foo }}
\n", + 12 + ], + "result": { + "index": 0, + "captureIndices": [ + { + "start": 12, + "end": 13, + "length": 1 + } + ] + } + }, { "args": [ "
{{ foo }}
\n", diff --git a/packages/shiki/test/grammar-state.test.ts b/packages/shiki/test/grammar-state.test.ts index a52cfd1a1..ec0b7b8db 100644 --- a/packages/shiki/test/grammar-state.test.ts +++ b/packages/shiki/test/grammar-state.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'vitest' -import { createHighlighter } from '../src' +import { createHighlighter, hastToHtml } from '../src' it('getLastGrammarState', async () => { const shiki = await createHighlighter({ @@ -19,6 +19,9 @@ it('getLastGrammarState', async () => { "source.ts", ], "theme": "vitesse-light", + "themes": [ + "vitesse-light", + ], } `) @@ -56,6 +59,16 @@ it('getLastGrammarState', async () => { { "bg": "#ffffff", "fg": "#393a34", + "grammarState": { + "lang": "typescript", + "scopes": [ + "source.ts", + ], + "theme": "vitesse-light", + "themes": [ + "vitesse-light", + ], + }, "rootStyle": undefined, "themeName": "vitesse-light", "tokens": [ @@ -120,53 +133,66 @@ it('getLastGrammarState', async () => { `) expect.soft(highlightedContext).toMatchInlineSnapshot(` - { - "bg": "#ffffff", - "fg": "#393a34", - "rootStyle": undefined, - "themeName": "vitesse-light", - "tokens": [ - [ - { - "color": "#2E8F82", - "content": "Omit", - "fontStyle": 0, - "offset": 0, - }, - { - "color": "#999999", - "content": "<{}, ", - "fontStyle": 0, - "offset": 4, - }, - { - "color": "#2E8F82", - "content": "string", - "fontStyle": 0, - "offset": 9, - }, - { - "color": "#999999", - "content": " | ", - "fontStyle": 0, - "offset": 15, - }, - { - "color": "#2E8F82", - "content": "number", - "fontStyle": 0, - "offset": 18, - }, - { - "color": "#999999", - "content": ">", - "fontStyle": 0, - "offset": 24, - }, - ], + { + "bg": "#ffffff", + "fg": "#393a34", + "grammarState": { + "lang": "typescript", + "scopes": [ + "meta.type.annotation.ts", + "meta.var-single-variable.expr.ts", + "meta.var.expr.ts", + "source.ts", ], - } - `) + "theme": "vitesse-light", + "themes": [ + "vitesse-light", + ], + }, + "rootStyle": undefined, + "themeName": "vitesse-light", + "tokens": [ + [ + { + "color": "#2E8F82", + "content": "Omit", + "fontStyle": 0, + "offset": 0, + }, + { + "color": "#999999", + "content": "<{}, ", + "fontStyle": 0, + "offset": 4, + }, + { + "color": "#2E8F82", + "content": "string", + "fontStyle": 0, + "offset": 9, + }, + { + "color": "#999999", + "content": " | ", + "fontStyle": 0, + "offset": 15, + }, + { + "color": "#2E8F82", + "content": "number", + "fontStyle": 0, + "offset": 18, + }, + { + "color": "#999999", + "content": ">", + "fontStyle": 0, + "offset": 24, + }, + ], + ], + } + `) }) it('grammarContextCode', async () => { @@ -210,6 +236,86 @@ it('grammarContextCode', async () => { .toEqual(highlightedVueBare) }) +it('getLastGrammarState with multiple themes', async () => { + const shiki = await createHighlighter({ + themes: ['vitesse-light', 'vitesse-dark'], + langs: ['typescript'], + }) + + const tokens = shiki.codeToTokens('let a:', { + lang: 'typescript', + themes: { + light: 'vitesse-light', + dark: 'vitesse-dark', + }, + }) + + expect(tokens.grammarState).toBeDefined() + + const input = 'Omit<{}, string | number>' + + const highlightedWithState = shiki.codeToHtml(input, { + lang: 'typescript', + themes: { + light: 'vitesse-light', + dark: 'vitesse-dark', + }, + grammarState: tokens.grammarState, + }) + + const highlightedWithoutState = shiki.codeToHtml(input, { + lang: 'typescript', + themes: { + light: 'vitesse-light', + dark: 'vitesse-dark', + }, + }) + + expect(highlightedWithoutState) + .not + .toEqual(highlightedWithState) + + const highlightedWithSingleTheme = shiki.codeToHtml(input, { + lang: 'typescript', + theme: 'vitesse-light', + grammarState: tokens.grammarState, + }) + + expect(highlightedWithSingleTheme).toBeDefined() +}) + +it('getLastGrammarState from hast', async () => { + const shiki = await createHighlighter({ + themes: ['vitesse-light'], + langs: ['typescript'], + }) + + const part1 = 'let a = "' + const part2 = 'console.log(a)"' + + const highlightedFull = shiki.codeToHast(part1 + part2, { + lang: 'typescript', + theme: 'vitesse-light', + }) + + const highlightedPart1 = shiki.codeToHast(part1, { + lang: 'typescript', + theme: 'vitesse-light', + }) + + const state = shiki.getLastGrammarState(highlightedPart1) + expect(state).toBeDefined() + + const highlighted = shiki.codeToHtml(part2, { + lang: 'typescript', + theme: 'vitesse-light', + grammarState: state, + }) + + expect(hastToHtml(highlightedFull)).toContain('console.log') + expect(highlighted).toContain('console.log') +}) + describe('errors', () => { it('should throw on wrong language', async () => { const shiki = await createHighlighter({ @@ -247,6 +353,6 @@ describe('errors', () => { theme: 'vitesse-dark', grammarState: state, })) - .toThrowErrorMatchingInlineSnapshot(`[ShikiError: Grammar state theme "vitesse-light" does not match highlight theme "vitesse-dark"]`) + .toThrowErrorMatchingInlineSnapshot(`[ShikiError: Grammar state themes "vitesse-light" do not contain highlight theme "vitesse-dark"]`) }) }) diff --git a/packages/types/src/highlighter.ts b/packages/types/src/highlighter.ts index 73b97724e..d91886ba3 100644 --- a/packages/types/src/highlighter.ts +++ b/packages/types/src/highlighter.ts @@ -15,6 +15,11 @@ import type { } from './tokens' import type { Awaitable, MaybeArray } from './utils' +/** + * Type of object that can be bound to a grammar state + */ +export type GrammarStateMapKey = Root | ThemedToken[][] + /** * Internal context of Shiki, core textmate logic */ @@ -127,10 +132,10 @@ export interface HighlighterGeneric, ResolveBundleKey> - ) => GrammarState + getLastGrammarState: { + (element: GrammarStateMapKey, options?: never): GrammarState | undefined + (code: string, options: CodeToTokensBaseOptions, ResolveBundleKey>): GrammarState + } /** * Get internal context object diff --git a/packages/types/src/tokens.ts b/packages/types/src/tokens.ts index 43f198d23..2453dd802 100644 --- a/packages/types/src/tokens.ts +++ b/packages/types/src/tokens.ts @@ -1,4 +1,4 @@ -import type { FontStyle, IRawThemeSetting } from '@shikijs/vscode-textmate' +import type { FontStyle, IRawThemeSetting, StateStack } from '@shikijs/vscode-textmate' import type { SpecialLanguage } from './langs' import type { CodeOptionsThemes } from './options' import type { SpecialTheme, ThemeRegistrationAny } from './themes' @@ -11,6 +11,15 @@ import type { SpecialTheme, ThemeRegistrationAny } from './themes' export interface GrammarState { readonly lang: string readonly theme: string + readonly themes: string[] + /** + * @internal + */ + getInternalStack: (theme?: string) => StateStack | undefined + getScopes: (theme?: string) => string[] | undefined + /** + * @deprecated Use `getScopes` instead. + */ get scopes(): string[] } @@ -238,4 +247,9 @@ export interface TokensResult { * When specified, `fg` and `bg` will be ignored. */ rootStyle?: string + + /** + * The last grammar state of the code snippet. + */ + grammarState?: GrammarState }