From 90a036ffa66c98806afd176bda78709543d02baa Mon Sep 17 00:00:00 2001 From: Benoit Daloze Date: Wed, 28 Aug 2024 22:32:30 +0200 Subject: [PATCH] Type every node field and mark on-error-only types explicitly * For Loader.java, do not deserialize the AST if there are errors, so then Java nodes only have non-error types for fields. --- config.yml | 216 +++++++++++++++++++++-- rakelib/typecheck.rake | 1 + rust/ruby-prism/build.rs | 61 +++++-- src/prism.c | 33 +++- templates/include/prism/ast.h.erb | 2 +- templates/java/org/prism/Loader.java.erb | 21 ++- templates/lib/prism/dsl.rb.erb | 4 +- templates/lib/prism/node.rb.erb | 2 +- templates/rbi/prism/dsl.rbi.erb | 4 +- templates/template.rb | 39 +++- 10 files changed, 326 insertions(+), 57 deletions(-) diff --git a/config.yml b/config.yml index dc19295c3e7..957035b6545 100644 --- a/config.yml +++ b/config.yml @@ -789,8 +789,8 @@ nodes: - GlobalVariableReadNode - BackReferenceReadNode - NumberedReferenceReadNode - - SymbolNode # On parsing error of `alias $a b` - - MissingNode # On parsing error of `alias $a 42` + - on error: SymbolNode # alias $a b + - on error: MissingNode # alias $a 42 comment: | Represents the old name of the global variable that can be used before aliasing. @@ -820,8 +820,8 @@ nodes: kind: - SymbolNode - InterpolatedSymbolNode - - GlobalVariableReadNode # On parsing error of `alias a $b` - - MissingNode # On parsing error of `alias a 42` + - on error: GlobalVariableReadNode # alias a $b + - on error: MissingNode # alias a 42 - name: keyword_loc type: location comment: | @@ -833,8 +833,10 @@ nodes: fields: - name: left type: node + kind: pattern expression - name: right type: node + kind: pattern expression - name: operator_loc type: location comment: | @@ -846,6 +848,7 @@ nodes: fields: - name: left type: node + kind: non-void expression comment: | Represents the left side of the expression. It can be any [non-void expression](https://github.com/ruby/prism/blob/main/docs/parsing_rules.md#non-void-expression). @@ -856,6 +859,7 @@ nodes: ^ - name: right type: node + kind: non-void expression comment: | Represents the right side of the expression. It can be any [non-void expression](https://github.com/ruby/prism/blob/main/docs/parsing_rules.md#non-void-expression). @@ -881,6 +885,7 @@ nodes: fields: - name: arguments type: node[] + kind: non-void expression comment: | Represents a set of arguments to a method or a keyword. @@ -891,6 +896,7 @@ nodes: fields: - name: elements type: node[] + kind: non-void expression comment: Represent the list of zero or more [non-void expressions](https://github.com/ruby/prism/blob/main/docs/parsing_rules.md#non-void-expression) within the array. - name: opening_loc type: location? @@ -919,12 +925,18 @@ nodes: fields: - name: constant type: node? + kind: + - ConstantReadNode + - ConstantPathNode - name: requireds type: node[] + kind: pattern expression - name: rest type: node? + kind: pattern expression - name: posts type: node[] + kind: pattern expression - name: opening_loc type: location? - name: closing_loc @@ -950,6 +962,7 @@ nodes: fields: - name: key type: node + kind: non-void expression comment: | The key of the association. This can be any [non-void expression](https://github.com/ruby/prism/blob/main/docs/parsing_rules.md#non-void-expression). @@ -963,6 +976,7 @@ nodes: ^^^^^^^^^^ - name: value type: node + kind: non-void expression comment: | The value of the association, if present. This can be any [non-void expression](https://github.com/ruby/prism/blob/main/docs/parsing_rules.md#non-void-expression). @@ -987,6 +1001,7 @@ nodes: fields: - name: value type: node? + kind: non-void expression comment: | The value to be splatted, if present. Will be missing when keyword rest argument forwarding is used. @@ -1049,6 +1064,7 @@ nodes: fields: - name: expression type: node? + kind: non-void expression - name: operator_loc type: location comment: | @@ -1153,6 +1169,7 @@ nodes: fields: - name: receiver type: node? + kind: non-void expression - name: call_operator_loc type: location? - name: message_loc @@ -1165,6 +1182,7 @@ nodes: type: location - name: value type: node + kind: non-void expression comment: | Represents the use of the `&&=` operator on a call. @@ -1175,6 +1193,7 @@ nodes: fields: - name: receiver type: node? + kind: non-void expression comment: | The object that the method is being called on. This can be either `nil` or any [non-void expression](https://github.com/ruby/prism/blob/main/docs/parsing_rules.md#non-void-expression). @@ -1201,6 +1220,9 @@ nodes: type: location? - name: block type: node? + kind: + - BlockNode + - BlockArgumentNode comment: | Represents a method call, in all of the various forms that can take. @@ -1226,6 +1248,7 @@ nodes: fields: - name: receiver type: node? + kind: non-void expression - name: call_operator_loc type: location? - name: message_loc @@ -1240,6 +1263,7 @@ nodes: type: location - name: value type: node + kind: non-void expression comment: | Represents the use of an assignment operator on a call. @@ -1250,6 +1274,7 @@ nodes: fields: - name: receiver type: node? + kind: non-void expression - name: call_operator_loc type: location? - name: message_loc @@ -1262,6 +1287,7 @@ nodes: type: location - name: value type: node + kind: non-void expression comment: | Represents the use of the `||=` operator on a call. @@ -1272,6 +1298,7 @@ nodes: fields: - name: receiver type: node + kind: non-void expression - name: call_operator_loc type: location - name: name @@ -1295,8 +1322,10 @@ nodes: fields: - name: value type: node + kind: pattern expression - name: target type: node + kind: LocalVariableTargetNode - name: operator_loc type: location comment: | @@ -1308,8 +1337,10 @@ nodes: fields: - name: predicate type: node? + kind: non-void expression - name: conditions type: node[] + kind: InNode - name: else_clause type: node? kind: ElseNode @@ -1328,8 +1359,10 @@ nodes: fields: - name: predicate type: node? + kind: non-void expression - name: conditions type: node[] + kind: WhenNode - name: else_clause type: node? kind: ElseNode @@ -1352,12 +1385,20 @@ nodes: type: location - name: constant_path type: node + kind: + - ConstantReadNode + - ConstantPathNode + - on error: CallNode # class 0.X end - name: inheritance_operator_loc type: location? - name: superclass type: node? + kind: non-void expression - name: body type: node? + kind: + - StatementsNode + - BeginNode - name: end_keyword_loc type: location - name: name @@ -1377,6 +1418,7 @@ nodes: type: location - name: value type: node + kind: non-void expression comment: | Represents the use of the `&&=` operator for assignment to a class variable. @@ -1392,6 +1434,7 @@ nodes: type: location - name: value type: node + kind: non-void expression - name: binary_operator type: constant comment: | @@ -1409,6 +1452,7 @@ nodes: type: location - name: value type: node + kind: non-void expression comment: | Represents the use of the `||=` operator for assignment to a class variable. @@ -1457,6 +1501,7 @@ nodes: ^^^^^ - name: value type: node + kind: non-void expression comment: | The value to write to the class variable. This can be any [non-void expression](https://github.com/ruby/prism/blob/main/docs/parsing_rules.md#non-void-expression). @@ -1487,6 +1532,7 @@ nodes: type: location - name: value type: node + kind: non-void expression comment: | Represents the use of the `&&=` operator for assignment to a constant. @@ -1502,6 +1548,7 @@ nodes: type: location - name: value type: node + kind: non-void expression - name: binary_operator type: constant comment: | @@ -1519,6 +1566,7 @@ nodes: type: location - name: value type: node + kind: non-void expression comment: | Represents the use of the `||=` operator for assignment to a constant. @@ -1533,6 +1581,7 @@ nodes: type: location - name: value type: node + kind: non-void expression comment: | Represents the use of the `&&=` operator for assignment to a constant path. @@ -1542,6 +1591,7 @@ nodes: fields: - name: parent type: node? + kind: non-void expression comment: | The left-hand node of the path, if present. It can be `nil` or any [non-void expression](https://github.com/ruby/prism/blob/main/docs/parsing_rules.md#non-void-expression). It will be `nil` when the constant lookup is at the root of the module tree. @@ -1590,6 +1640,7 @@ nodes: type: location - name: value type: node + kind: non-void expression - name: binary_operator type: constant comment: | @@ -1606,6 +1657,7 @@ nodes: type: location - name: value type: node + kind: non-void expression comment: | Represents the use of the `||=` operator for assignment to a constant path. @@ -1615,6 +1667,7 @@ nodes: fields: - name: parent type: node? + kind: non-void expression - name: name type: constant? - name: delimiter_loc @@ -1648,6 +1701,7 @@ nodes: ^ - name: value type: node + kind: non-void expression comment: | The value to write to the constant path. It can be any [non-void expression](https://github.com/ruby/prism/blob/main/docs/parsing_rules.md#non-void-expression). @@ -1707,6 +1761,7 @@ nodes: ^^^ - name: value type: node + kind: non-void expression comment: | The value to write to the constant. It can be any [non-void expression](https://github.com/ruby/prism/blob/main/docs/parsing_rules.md#non-void-expression). @@ -1735,11 +1790,15 @@ nodes: type: location - name: receiver type: node? + kind: non-void expression - name: parameters type: node? kind: ParametersNode - name: body type: node? + kind: + - StatementsNode + - BeginNode - name: locals type: constant[] - name: def_keyword_loc @@ -1766,6 +1825,7 @@ nodes: type: location? - name: value type: node + kind: Node # More than non-void expression as defined?(return) is allowed, yet defined?(BEGIN{}) is SyntaxError - name: rparen_loc type: location? - name: keyword_loc @@ -1809,6 +1869,12 @@ nodes: type: location - name: variable type: node + kind: + - InstanceVariableReadNode + - ClassVariableReadNode + - GlobalVariableReadNode + - BackReferenceReadNode + - NumberedReferenceReadNode comment: | Represents an interpolated variable. @@ -1842,12 +1908,20 @@ nodes: fields: - name: constant type: node? + kind: + - ConstantReadNode + - ConstantPathNode - name: left type: node + kind: SplatNode - name: requireds type: node[] + kind: pattern expression - name: right type: node + kind: + - SplatNode + - on error: MissingNode - name: opening_loc type: location? - name: closing_loc @@ -1868,8 +1942,10 @@ nodes: fields: - name: left type: node? + kind: non-void expression - name: right type: node? + kind: non-void expression - name: operator_loc type: location comment: | @@ -1891,6 +1967,19 @@ nodes: fields: - name: index type: node + kind: + - LocalVariableTargetNode + - InstanceVariableTargetNode + - ClassVariableTargetNode + - GlobalVariableTargetNode + - ConstantTargetNode + - ConstantPathTargetNode + - CallTargetNode + - IndexTargetNode + - MultiTargetNode + - on error: BackReferenceReadNode # for $& in a end + - on error: NumberedReferenceReadNode # for $1 in a end + - on error: MissingNode # for in 1..10; end comment: | The index expression for `for` loops. @@ -1898,6 +1987,7 @@ nodes: ^ - name: collection type: node + kind: non-void expression comment: | The collection to iterate over. @@ -1981,6 +2071,7 @@ nodes: type: location - name: value type: node + kind: non-void expression comment: | Represents the use of the `&&=` operator for assignment to a global variable. @@ -1996,6 +2087,7 @@ nodes: type: location - name: value type: node + kind: non-void expression - name: binary_operator type: constant comment: | @@ -2013,6 +2105,7 @@ nodes: type: location - name: value type: node + kind: non-void expression comment: | Represents the use of the `||=` operator for assignment to a global variable. @@ -2061,6 +2154,7 @@ nodes: ^^^^ - name: value type: node + kind: non-void expression comment: | The value to write to the global variable. It can be any [non-void expression](https://github.com/ruby/prism/blob/main/docs/parsing_rules.md#non-void-expression). @@ -2119,6 +2213,9 @@ nodes: fields: - name: constant type: node? + kind: + - ConstantReadNode + - ConstantPathNode - name: elements type: node[] kind: AssocNode @@ -2152,6 +2249,7 @@ nodes: The `if_keyword_loc` field will be `nil` when the `IfNode` represents a ternary expression. - name: predicate type: node + kind: non-void expression comment: | The node for the condition the `IfNode` is testing. @@ -2244,6 +2342,11 @@ nodes: fields: - name: value type: node + kind: + - LocalVariableReadNode + - CallNode + - ConstantReadNode + - LocalVariableTargetNode comment: | Represents a node that is implicitly being added to the tree but doesn't correspond directly to a node in the source. @@ -2274,6 +2377,7 @@ nodes: fields: - name: pattern type: node + kind: pattern expression - name: statements type: node? kind: StatementsNode @@ -2291,6 +2395,7 @@ nodes: fields: - name: receiver type: node? + kind: non-void expression - name: call_operator_loc type: location? - name: opening_loc @@ -2302,10 +2407,12 @@ nodes: type: location - name: block type: node? + kind: BlockArgumentNode # foo[&b] &&= value, only valid on Ruby < 3.4 - name: operator_loc type: location - name: value type: node + kind: non-void expression comment: | Represents the use of the `&&=` operator on a call to the `[]` method. @@ -2316,6 +2423,7 @@ nodes: fields: - name: receiver type: node? + kind: non-void expression - name: call_operator_loc type: location? - name: opening_loc @@ -2327,12 +2435,14 @@ nodes: type: location - name: block type: node? + kind: BlockArgumentNode # foo[&b] += value, only valid on Ruby < 3.4 - name: binary_operator type: constant - name: binary_operator_loc type: location - name: value type: node + kind: non-void expression comment: | Represents the use of an assignment operator on a call to `[]`. @@ -2343,6 +2453,7 @@ nodes: fields: - name: receiver type: node? + kind: non-void expression - name: call_operator_loc type: location? - name: opening_loc @@ -2354,10 +2465,12 @@ nodes: type: location - name: block type: node? + kind: BlockArgumentNode # foo[&b] ||= value, only valid on Ruby < 3.4 - name: operator_loc type: location - name: value type: node + kind: non-void expression comment: | Represents the use of the `||=` operator on a call to `[]`. @@ -2368,6 +2481,7 @@ nodes: fields: - name: receiver type: node + kind: non-void expression - name: opening_loc type: location - name: arguments @@ -2377,6 +2491,7 @@ nodes: type: location - name: block type: node? + kind: BlockArgumentNode # foo[&b], = 1, only valid on Ruby < 3.4 comment: | Represents assigning to an index. @@ -2400,6 +2515,7 @@ nodes: type: location - name: value type: node + kind: non-void expression comment: | Represents the use of the `&&=` operator for assignment to an instance variable. @@ -2415,6 +2531,7 @@ nodes: type: location - name: value type: node + kind: non-void expression - name: binary_operator type: constant comment: | @@ -2432,6 +2549,7 @@ nodes: type: location - name: value type: node + kind: non-void expression comment: | Represents the use of the `||=` operator for assignment to an instance variable. @@ -2480,6 +2598,7 @@ nodes: ^^^ - name: value type: node + kind: non-void expression comment: | The value to write to the instance variable. It can be any [non-void expression](https://github.com/ruby/prism/blob/main/docs/parsing_rules.md#non-void-expression). @@ -2657,8 +2776,15 @@ nodes: type: location - name: parameters type: node? + kind: + - BlockParametersNode + - NumberedParametersNode + - ItParametersNode - name: body type: node? + kind: + - StatementsNode + - BeginNode comment: | Represents using a lambda literal (not the lambda method call). @@ -2672,6 +2798,7 @@ nodes: type: location - name: value type: node + kind: non-void expression - name: name type: constant - name: depth @@ -2689,6 +2816,7 @@ nodes: type: location - name: value type: node + kind: non-void expression - name: name type: constant - name: binary_operator @@ -2708,6 +2836,7 @@ nodes: type: location - name: value type: node + kind: non-void expression - name: name type: constant - name: depth @@ -2787,6 +2916,7 @@ nodes: ^^^ - name: value type: node + kind: non-void expression comment: | The value to write to the local variable. It can be any [non-void expression](https://github.com/ruby/prism/blob/main/docs/parsing_rules.md#non-void-expression). @@ -2831,8 +2961,10 @@ nodes: fields: - name: value type: node + kind: non-void expression - name: pattern type: node + kind: pattern expression - name: operator_loc type: location comment: | @@ -2844,8 +2976,10 @@ nodes: fields: - name: value type: node + kind: non-void expression - name: pattern type: node + kind: pattern expression - name: operator_loc type: location comment: | @@ -2877,8 +3011,15 @@ nodes: type: location - name: constant_path type: node + kind: + - ConstantReadNode + - ConstantPathNode + - on error: MissingNode # module Parent module end - name: body type: node? + kind: + - StatementsNode + - BeginNode - name: end_keyword_loc type: location - name: name @@ -2902,9 +3043,9 @@ nodes: - CallTargetNode - IndexTargetNode - MultiTargetNode - - RequiredParameterNode - - BackReferenceReadNode # On parsing error of `$',` - - NumberedReferenceReadNode # On parsing error of `$1,` + - RequiredParameterNode # def m((a,b)); end + - on error: BackReferenceReadNode # a, (b, $&) = z + - on error: NumberedReferenceReadNode # a, (b, $1) = z comment: | Represents the targets expressions before a splat node. @@ -2947,8 +3088,9 @@ nodes: - CallTargetNode - IndexTargetNode - MultiTargetNode - - RequiredParameterNode - - BackReferenceReadNode # On parsing error of `*,$'` + - RequiredParameterNode # def m((*,b)); end + - on error: BackReferenceReadNode # a, (*, $&) = z + - on error: NumberedReferenceReadNode # a, (*, $1) = z comment: | Represents the targets expressions after a splat node. @@ -2992,6 +3134,8 @@ nodes: - CallTargetNode - IndexTargetNode - MultiTargetNode + - on error: BackReferenceReadNode # $&, = z + - on error: NumberedReferenceReadNode # $1, = z comment: | Represents the targets expressions before a splat node. @@ -3034,6 +3178,8 @@ nodes: - CallTargetNode - IndexTargetNode - MultiTargetNode + - on error: BackReferenceReadNode # *, $& = z + - on error: NumberedReferenceReadNode # *, $1 = z comment: | Represents the targets expressions after a splat node. @@ -3062,6 +3208,7 @@ nodes: ^ - name: value type: node + kind: non-void expression comment: | The value to write to the targets. It can be any [non-void expression](https://github.com/ruby/prism/blob/main/docs/parsing_rules.md#non-void-expression). @@ -3137,6 +3284,7 @@ nodes: type: location - name: value type: node + kind: non-void expression comment: | Represents an optional keyword parameter to a method, block, or lambda definition. @@ -3154,6 +3302,7 @@ nodes: type: location - name: value type: node + kind: non-void expression comment: | Represents an optional parameter to a method, block, or lambda definition. @@ -3164,6 +3313,7 @@ nodes: fields: - name: left type: node + kind: non-void expression comment: | Represents the left side of the expression. It can be any [non-void expression](https://github.com/ruby/prism/blob/main/docs/parsing_rules.md#non-void-expression). @@ -3174,6 +3324,7 @@ nodes: ^ - name: right type: node + kind: non-void expression comment: | Represents the right side of the expression. It can be any [non-void expression](https://github.com/ruby/prism/blob/main/docs/parsing_rules.md#non-void-expression). @@ -3215,10 +3366,10 @@ nodes: - RequiredParameterNode - MultiTargetNode # On parsing error of `f(**kwargs, ...)` or `f(**nil, ...)`, the keyword_rest value is moved here: - - KeywordRestParameterNode - - NoKeywordsParameterNode + - on error: KeywordRestParameterNode + - on error: NoKeywordsParameterNode # On parsing error of `f(..., ...)`, the first forwarding parameter is moved here: - - ForwardingParameterNode + - on error: ForwardingParameterNode - name: keywords type: node[] kind: @@ -3243,6 +3394,7 @@ nodes: fields: - name: body type: node? + kind: non-void expression # Usually a StatementsNode but not always e.g. `1 in (..10)` - name: opening_loc type: location - name: closing_loc @@ -3257,6 +3409,7 @@ nodes: fields: - name: expression type: node + kind: non-void expression - name: operator_loc type: location - name: lparen_loc @@ -3272,6 +3425,15 @@ nodes: fields: - name: variable type: node + kind: + - LocalVariableReadNode + - InstanceVariableReadNode + - ClassVariableReadNode + - GlobalVariableReadNode # foo in ^$a + - BackReferenceReadNode # foo in ^$& + - NumberedReferenceReadNode # foo in ^$1 + - ItLocalVariableReadNode # proc { 1 in ^it } + - on error: MissingNode # foo in ^Bar - name: operator_loc type: location comment: | @@ -3324,6 +3486,7 @@ nodes: fields: - name: left type: node? + kind: non-void expression comment: | The left-hand side of the range, if present. It can be either `nil` or any [non-void expression](https://github.com/ruby/prism/blob/main/docs/parsing_rules.md#non-void-expression). @@ -3334,6 +3497,7 @@ nodes: ^^^^^ - name: right type: node? + kind: non-void expression comment: | The right-hand side of the range, if present. It can be either `nil` or any [non-void expression](https://github.com/ruby/prism/blob/main/docs/parsing_rules.md#non-void-expression). @@ -3425,10 +3589,12 @@ nodes: fields: - name: expression type: node + kind: non-void expression - name: keyword_loc type: location - name: rescue_expression type: node + kind: non-void expression newline: expression comment: | Represents an expression modified with a rescue. @@ -3441,10 +3607,23 @@ nodes: type: location - name: exceptions type: node[] + kind: non-void expression - name: operator_loc type: location? - name: reference type: node? + kind: + - LocalVariableTargetNode + - InstanceVariableTargetNode + - ClassVariableTargetNode + - GlobalVariableTargetNode + - ConstantTargetNode + - ConstantPathTargetNode + - CallTargetNode + - IndexTargetNode + - on error: BackReferenceReadNode # => begin; rescue => $&; end + - on error: NumberedReferenceReadNode # => begin; rescue => $1; end + - on error: MissingNode # begin; rescue =>; end - name: statements type: node? kind: StatementsNode @@ -3531,8 +3710,12 @@ nodes: type: location - name: expression type: node + kind: non-void expression - name: body type: node? + kind: + - StatementsNode + - BeginNode - name: end_keyword_loc type: location comment: | @@ -3569,6 +3752,7 @@ nodes: type: location - name: expression type: node? + kind: non-void expression comment: | Represents the use of the splat operator. @@ -3578,6 +3762,7 @@ nodes: fields: - name: body type: node[] + kind: Node # fun fact: the only place where absolutely every node is valid comment: | Represents a set of statements contained within some scope. @@ -3618,6 +3803,9 @@ nodes: type: location? - name: block type: node? + kind: + - BlockNode + - BlockArgumentNode comment: | Represents the use of the `super` keyword with parentheses or arguments. @@ -3679,6 +3867,7 @@ nodes: ^^^^^^ - name: predicate type: node + kind: non-void expression comment: | The condition to be evaluated for the unless expression. It can be any [non-void expression](https://github.com/ruby/prism/blob/main/docs/parsing_rules.md#non-void-expression). @@ -3736,6 +3925,7 @@ nodes: type: location? - name: predicate type: node + kind: non-void expression - name: statements type: node? kind: StatementsNode @@ -3754,6 +3944,7 @@ nodes: type: location - name: conditions type: node[] + kind: non-void expression - name: then_keyword_loc type: location? - name: statements @@ -3775,6 +3966,7 @@ nodes: type: location? - name: predicate type: node + kind: non-void expression - name: statements type: node? kind: StatementsNode diff --git a/rakelib/typecheck.rake b/rakelib/typecheck.rake index 3e2aeb9a585..052a20327e5 100644 --- a/rakelib/typecheck.rake +++ b/rakelib/typecheck.rake @@ -59,6 +59,7 @@ namespace :typecheck do --ignore=test/ --ignore=rakelib/ --ignore=Rakefile + --ignore=top-100-gems/ # Treat all files as "typed: true" by default --typed=true # Use the typed-override file to revert some files to "typed: false" diff --git a/rust/ruby-prism/build.rs b/rust/ruby-prism/build.rs index c8618815c51..9ca1eb0971a 100644 --- a/rust/ruby-prism/build.rs +++ b/rust/ruby-prism/build.rs @@ -47,11 +47,27 @@ enum NodeFieldType { Double, } +#[derive(Debug, Deserialize)] +#[allow(dead_code)] +struct OnErrorType { + #[serde(rename = "on error")] + kind: String, +} + +#[derive(Debug, Deserialize)] +#[serde(untagged)] +#[allow(dead_code)] +enum UnionKind { + OnSuccess(String), + OnError(OnErrorType), +} + #[derive(Debug, Deserialize)] #[serde(untagged)] +#[allow(dead_code)] enum NodeFieldKind { Concrete(String), - Union(Vec), + Union(Vec), } #[derive(Debug, Deserialize)] @@ -126,6 +142,13 @@ fn struct_name(name: &str) -> String { result } +fn kind_to_type(kind: &String) -> String { + match kind.as_str() { + "non-void expression" | "pattern expression" | "Node" => String::from(""), + _ => kind.to_string(), + } +} + /// Returns the name of the C type from the given node name. fn type_name(name: &str) -> String { let mut result = String::with_capacity(8 + name.len()); @@ -263,30 +286,34 @@ fn write_node(file: &mut File, flags: &[Flags], node: &Node) -> Result<(), Box { - if let Some(NodeFieldKind::Concrete(kind)) = &field.kind { + NodeFieldType::Node => match &field.kind { + Some(NodeFieldKind::Concrete(raw_kind)) if kind_to_type(raw_kind) != "" => { + let kind = kind_to_type(raw_kind); writeln!(file, " pub fn {}(&self) -> {}<'pr> {{", field.name, kind)?; - writeln!(file, " let node: *mut pm{}_t = unsafe {{ (*self.pointer).{} }};", struct_name(kind), field.name)?; + writeln!(file, " let node: *mut pm{}_t = unsafe {{ (*self.pointer).{} }};", struct_name(&kind), field.name)?; writeln!(file, " {} {{ parser: self.parser, pointer: node, marker: PhantomData }}", kind)?; writeln!(file, " }}")?; - } else { + }, + _ => { writeln!(file, " pub fn {}(&self) -> Node<'pr> {{", field.name)?; writeln!(file, " let node: *mut pm_node_t = unsafe {{ (*self.pointer).{} }};", field.name)?; writeln!(file, " Node::new(self.parser, node)")?; writeln!(file, " }}")?; - } + }, }, - NodeFieldType::OptionalNode => { - if let Some(NodeFieldKind::Concrete(kind)) = &field.kind { + NodeFieldType::OptionalNode => match &field.kind { + Some(NodeFieldKind::Concrete(raw_kind)) if kind_to_type(raw_kind) != "" => { + let kind = kind_to_type(raw_kind); writeln!(file, " pub fn {}(&self) -> Option<{}<'pr>> {{", field.name, kind)?; - writeln!(file, " let node: *mut pm{}_t = unsafe {{ (*self.pointer).{} }};", struct_name(kind), field.name)?; + writeln!(file, " let node: *mut pm{}_t = unsafe {{ (*self.pointer).{} }};", struct_name(&kind), field.name)?; writeln!(file, " if node.is_null() {{")?; writeln!(file, " None")?; writeln!(file, " }} else {{")?; writeln!(file, " Some({} {{ parser: self.parser, pointer: node, marker: PhantomData }})", kind)?; writeln!(file, " }}")?; writeln!(file, " }}")?; - } else { + }, + _ => { writeln!(file, " pub fn {}(&self) -> Option> {{", field.name)?; writeln!(file, " let node: *mut pm_node_t = unsafe {{ (*self.pointer).{} }};", field.name)?; writeln!(file, " if node.is_null() {{")?; @@ -295,7 +322,7 @@ fn write_node(file: &mut File, flags: &[Flags], node: &Node) -> Result<(), Box { writeln!(file, " pub fn {}(&self) -> NodeList<'pr> {{", field.name)?; @@ -473,16 +500,18 @@ fn write_visit(file: &mut File, config: &Config) -> Result<(), Box { - if let Some(NodeFieldKind::Concrete(kind)) = &field.kind { - writeln!(file, " visitor.visit{}(&node.{}());", struct_name(kind), field.name)?; + if let Some(NodeFieldKind::Concrete(raw_kind)) = &field.kind { + let kind = kind_to_type(raw_kind); + writeln!(file, " visitor.visit{}(&node.{}());", struct_name(&kind), field.name)?; } else { writeln!(file, " visitor.visit(&node.{}());", field.name)?; } }, NodeFieldType::OptionalNode => { - if let Some(NodeFieldKind::Concrete(kind)) = &field.kind { + if let Some(NodeFieldKind::Concrete(raw_kind)) = &field.kind { + let kind = kind_to_type(raw_kind); writeln!(file, " if let Some(node) = node.{}() {{", field.name)?; - writeln!(file, " visitor.visit{}(&node);", struct_name(kind))?; + writeln!(file, " visitor.visit{}(&node);", struct_name(&kind))?; writeln!(file, " }}")?; } else { writeln!(file, " if let Some(node) = node.{}() {{", field.name)?; @@ -762,7 +791,7 @@ impl TryInto for Integer<'_> {{ let length = unsafe {{ (*self.pointer).length }}; if length == 0 {{ - i32::try_from(unsafe {{ (*self.pointer).value }}).map_or(Err(()), |value| + i32::try_from(unsafe {{ (*self.pointer).value }}).map_or(Err(()), |value| if negative {{ Ok(-value) }} else {{ diff --git a/src/prism.c b/src/prism.c index 28bd8d0d439..029d7be1d37 100644 --- a/src/prism.c +++ b/src/prism.c @@ -2985,6 +2985,7 @@ pm_index_and_write_node_create(pm_parser_t *parser, pm_call_node_t *target, cons pm_index_arguments_check(parser, target->arguments, target->block); + assert(!target->block || PM_NODE_TYPE_P(target->block, PM_BLOCK_ARGUMENT_NODE)); *node = (pm_index_and_write_node_t) { { .type = PM_INDEX_AND_WRITE_NODE, @@ -3000,7 +3001,7 @@ pm_index_and_write_node_create(pm_parser_t *parser, pm_call_node_t *target, cons .opening_loc = target->opening_loc, .arguments = target->arguments, .closing_loc = target->closing_loc, - .block = target->block, + .block = (pm_block_argument_node_t *) target->block, .operator_loc = PM_LOCATION_TOKEN_VALUE(operator), .value = value }; @@ -3060,6 +3061,7 @@ pm_index_operator_write_node_create(pm_parser_t *parser, pm_call_node_t *target, pm_index_arguments_check(parser, target->arguments, target->block); + assert(!target->block || PM_NODE_TYPE_P(target->block, PM_BLOCK_ARGUMENT_NODE)); *node = (pm_index_operator_write_node_t) { { .type = PM_INDEX_OPERATOR_WRITE_NODE, @@ -3075,7 +3077,7 @@ pm_index_operator_write_node_create(pm_parser_t *parser, pm_call_node_t *target, .opening_loc = target->opening_loc, .arguments = target->arguments, .closing_loc = target->closing_loc, - .block = target->block, + .block = (pm_block_argument_node_t *) target->block, .binary_operator = pm_parser_constant_id_location(parser, operator->start, operator->end - 1), .binary_operator_loc = PM_LOCATION_TOKEN_VALUE(operator), .value = value @@ -3137,6 +3139,7 @@ pm_index_or_write_node_create(pm_parser_t *parser, pm_call_node_t *target, const pm_index_arguments_check(parser, target->arguments, target->block); + assert(!target->block || PM_NODE_TYPE_P(target->block, PM_BLOCK_ARGUMENT_NODE)); *node = (pm_index_or_write_node_t) { { .type = PM_INDEX_OR_WRITE_NODE, @@ -3152,7 +3155,7 @@ pm_index_or_write_node_create(pm_parser_t *parser, pm_call_node_t *target, const .opening_loc = target->opening_loc, .arguments = target->arguments, .closing_loc = target->closing_loc, - .block = target->block, + .block = (pm_block_argument_node_t *) target->block, .operator_loc = PM_LOCATION_TOKEN_VALUE(operator), .value = value }; @@ -3205,6 +3208,7 @@ pm_index_target_node_create(pm_parser_t *parser, pm_call_node_t *target) { pm_index_arguments_check(parser, target->arguments, target->block); + assert(!target->block || PM_NODE_TYPE_P(target->block, PM_BLOCK_ARGUMENT_NODE)); *node = (pm_index_target_node_t) { { .type = PM_INDEX_TARGET_NODE, @@ -3216,7 +3220,7 @@ pm_index_target_node_create(pm_parser_t *parser, pm_call_node_t *target) { .opening_loc = target->opening_loc, .arguments = target->arguments, .closing_loc = target->closing_loc, - .block = target->block + .block = (pm_block_argument_node_t *) target->block, }; // Here we're going to free the target, since it is no longer necessary. @@ -3231,7 +3235,7 @@ pm_index_target_node_create(pm_parser_t *parser, pm_call_node_t *target) { * Allocate and initialize a new CapturePatternNode node. */ static pm_capture_pattern_node_t * -pm_capture_pattern_node_create(pm_parser_t *parser, pm_node_t *value, pm_node_t *target, const pm_token_t *operator) { +pm_capture_pattern_node_create(pm_parser_t *parser, pm_node_t *value, pm_local_variable_target_node_t *target, const pm_token_t *operator) { pm_capture_pattern_node_t *node = PM_NODE_ALLOC(parser, pm_capture_pattern_node_t); *node = (pm_capture_pattern_node_t) { @@ -3240,7 +3244,7 @@ pm_capture_pattern_node_create(pm_parser_t *parser, pm_node_t *value, pm_node_t .node_id = PM_NODE_IDENTIFY(parser), .location = { .start = value->location.start, - .end = target->location.end + .end = target->base.location.end }, }, .value = value, @@ -4032,14 +4036,25 @@ pm_find_pattern_node_create(pm_parser_t *parser, pm_node_list_t *nodes) { pm_find_pattern_node_t *node = PM_NODE_ALLOC(parser, pm_find_pattern_node_t); pm_node_t *left = nodes->nodes[0]; + assert(PM_NODE_TYPE_P(left, PM_SPLAT_NODE)); + pm_splat_node_t *left_splat_node = (pm_splat_node_t *) left; + pm_node_t *right; if (nodes->size == 1) { right = (pm_node_t *) pm_missing_node_create(parser, left->location.end, left->location.end); } else { right = nodes->nodes[nodes->size - 1]; + assert(PM_NODE_TYPE_P(right, PM_SPLAT_NODE)); } +#if PRISM_SERIALIZE_ONLY_SEMANTICS_FIELDS + // FindPatternNode#right is typed as SplatNode in this case, so replace the potential MissingNode with a SplatNode. + // The resulting AST will anyway be ignored, but this file still needs to compile. + pm_splat_node_t *right_splat_node = PM_NODE_TYPE_P(right, PM_SPLAT_NODE) ? (pm_splat_node_t *) right : left_splat_node; +#else + pm_node_t *right_splat_node = right; +#endif *node = (pm_find_pattern_node_t) { { .type = PM_FIND_PATTERN_NODE, @@ -4050,8 +4065,8 @@ pm_find_pattern_node_create(pm_parser_t *parser, pm_node_list_t *nodes) { }, }, .constant = NULL, - .left = left, - .right = right, + .left = left_splat_node, + .right = right_splat_node, .requireds = { 0 }, .opening_loc = PM_OPTIONAL_LOCATION_NOT_PROVIDED_VALUE, .closing_loc = PM_OPTIONAL_LOCATION_NOT_PROVIDED_VALUE @@ -17401,7 +17416,7 @@ parse_pattern_primitives(pm_parser_t *parser, pm_constant_id_list_t *captures, p } parse_pattern_capture(parser, captures, constant_id, &PM_LOCATION_TOKEN_VALUE(&parser->previous)); - pm_node_t *target = (pm_node_t *) pm_local_variable_target_node_create( + pm_local_variable_target_node_t *target = pm_local_variable_target_node_create( parser, &PM_LOCATION_TOKEN_VALUE(&parser->previous), constant_id, diff --git a/templates/include/prism/ast.h.erb b/templates/include/prism/ast.h.erb index 088a99323f8..c9b62306323 100644 --- a/templates/include/prism/ast.h.erb +++ b/templates/include/prism/ast.h.erb @@ -218,6 +218,6 @@ typedef enum pm_<%= flag.human %> { * to specify that through the environment. It will never be true except for in * those build systems. */ -#define PRISM_SERIALIZE_ONLY_SEMANTICS_FIELDS <%= Prism::Template::SERIALIZE_ONLY_SEMANTICS_FIELDS %> +#define PRISM_SERIALIZE_ONLY_SEMANTICS_FIELDS <%= Prism::Template::SERIALIZE_ONLY_SEMANTICS_FIELDS ? 1 : 0 %> #endif diff --git a/templates/java/org/prism/Loader.java.erb b/templates/java/org/prism/Loader.java.erb index bb978ab2c5e..af4d5bcec62 100644 --- a/templates/java/org/prism/Loader.java.erb +++ b/templates/java/org/prism/Loader.java.erb @@ -127,16 +127,21 @@ public class Loader { int constantPoolLength = loadVarUInt(); this.constantPool = new ConstantPool(this, source.bytes, constantPoolBufferOffset, constantPoolLength); - Nodes.Node node = loadNode(); + Nodes.Node node; + if (errors.length == 0) { + node = loadNode(); - int left = constantPoolBufferOffset - buffer.position(); - if (left != 0) { - throw new Error("Expected to consume all bytes while deserializing but there were " + left + " bytes left"); - } + int left = constantPoolBufferOffset - buffer.position(); + if (left != 0) { + throw new Error("Expected to consume all bytes while deserializing but there were " + left + " bytes left"); + } - boolean[] newlineMarked = new boolean[1 + source.getLineCount()]; - MarkNewlinesVisitor visitor = new MarkNewlinesVisitor(source, newlineMarked); - node.accept(visitor); + boolean[] newlineMarked = new boolean[1 + source.getLineCount()]; + MarkNewlinesVisitor visitor = new MarkNewlinesVisitor(source, newlineMarked); + node.accept(visitor); + } else { + node = null; + } return new ParseResult(node, magicComments, dataLocation, errors, warnings, source); } diff --git a/templates/lib/prism/dsl.rb.erb b/templates/lib/prism/dsl.rb.erb index f0dac2a4f0d..2a0732c66b0 100644 --- a/templates/lib/prism/dsl.rb.erb +++ b/templates/lib/prism/dsl.rb.erb @@ -70,10 +70,10 @@ module Prism def <%= node.human %>(<%= ["source: default_source", "node_id: 0", "location: default_location", "flags: 0", *node.fields.map { |field| case field when Prism::Template::NodeField - if !field.kind? + kind = field.specific_kind || field.union_kind.first + if kind == "Node" "#{field.name}: default_node(source, location)" else - kind = field.specific_kind || field.union_kind.first "#{field.name}: #{kind.gsub(/(?<=.)[A-Z]/, "_\\0").downcase}(source: source)" end when Prism::Template::ConstantField diff --git a/templates/lib/prism/node.rb.erb b/templates/lib/prism/node.rb.erb index 06a7212f61a..6d47cc887be 100644 --- a/templates/lib/prism/node.rb.erb +++ b/templates/lib/prism/node.rb.erb @@ -225,7 +225,7 @@ module Prism @flags = flags <%- node.fields.each do |field| -%> <%- if Prism::Template::CHECK_FIELD_KIND && field.respond_to?(:check_field_kind) -%> - raise <%= field.name %>.inspect unless <%= field.check_field_kind %> + raise "<%= node.name %>#<%= field.name %> was of unexpected type:\n#{<%= field.name %>.inspect}" unless <%= field.check_field_kind %> <%- end -%> @<%= field.name %> = <%= field.name %> <%- end -%> diff --git a/templates/rbi/prism/dsl.rbi.erb b/templates/rbi/prism/dsl.rbi.erb index 4f31af3710e..c5303855acb 100644 --- a/templates/rbi/prism/dsl.rbi.erb +++ b/templates/rbi/prism/dsl.rbi.erb @@ -14,10 +14,10 @@ module Prism::DSL ].concat(node.fields.map { |field| case field when Prism::Template::NodeField - if !field.kind? + kind = field.specific_kind || field.union_kind.first + if kind == "Node" [field.name, "default_node(source, location)", field.rbi_class] else - kind = field.specific_kind || field.union_kind.first [field.name, %Q{#{kind.gsub(/(?<=.)[A-Z]/, "_\\0").downcase}(source: source)}, field.rbi_class] end when Prism::Template::OptionalNodeField diff --git a/templates/template.rb b/templates/template.rb index 130e15d075b..ef3b7aa1a24 100755 --- a/templates/template.rb +++ b/templates/template.rb @@ -1,5 +1,5 @@ #!/usr/bin/env ruby -# typed: false +# typed: ignore require "erb" require "fileutils" @@ -8,6 +8,7 @@ module Prism module Template SERIALIZE_ONLY_SEMANTICS_FIELDS = ENV.fetch("PRISM_SERIALIZE_ONLY_SEMANTICS_FIELDS", false) + REMOVE_ON_ERROR_TYPES = SERIALIZE_ONLY_SEMANTICS_FIELDS CHECK_FIELD_KIND = ENV.fetch("CHECK_FIELD_KIND", false) JAVA_BACKEND = ENV["PRISM_JAVA_BACKEND"] || "truffleruby" @@ -93,8 +94,9 @@ def should_be_serialized? # Some node fields can be specialized if they point to a specific kind of # node and not just a generic node. class NodeKindField < Field - def kind? - options.key?(:kind) + def initialize(kind:, **options) + @kind = kind + super(**options) end def c_type @@ -115,18 +117,18 @@ def java_type def java_cast if specific_kind - "(Nodes.#{options[:kind]}) " + "(Nodes.#{@kind}) " else "" end end def specific_kind - options[:kind] unless options[:kind].is_a?(Array) + @kind unless @kind.is_a?(Array) end def union_kind - options[:kind] if options[:kind].is_a?(Array) + @kind if @kind.is_a?(Array) end end @@ -417,6 +419,31 @@ def initialize(config, flags) # changed to use fetch instead of delete. comment = options.delete(:comment) + if kinds = options[:kind] + kinds = [kinds] unless kinds.is_a?(Array) + kinds = kinds.map do |kind| + case kind + when "non-void expression" + # the actual list of types would be way too long + "Node" + when "pattern expression" + # TODO: can we be more precise, is the list small enough? + "Node" + when Hash + kind = kind.fetch("on error") + REMOVE_ON_ERROR_TYPES ? nil : kind + else + kind + end + end.compact + kinds = kinds.first if kinds.size == 1 + options[:kind] = kinds + else + if type < NodeKindField + raise "Missing kind in config.yml for field #{@name}##{options.fetch(:name)}" + end + end + type.new(comment: comment, **options) end