Use pl.lit to simplify types

signature: pl.lit(any): pl.Expr This is incredibly useful, because now the polars operator evaluators don't have to deal with InternalValueRepresentation
jvalue · May 29, 2024 · e7dbc05 · e7dbc05
1 parent be6d767
commit e7dbc05
Show file tree

Hide file tree

Showing 27 changed files with 170 additions and 344 deletions.
diff --git a/example/testing.jv b/example/testing.jv
@@ -0,0 +1,69 @@
+pipeline CarsPipeline {
+
+  CarsExtractor
+    -> CarsTextFileInterpreter
+    -> CarsCSVInterpreter
+    -> NameHeaderWriter
+    -> CarsTableInterpreter
+    -> MyTableTransformer
+    -> CarsLoader;
+
+
+  block CarsExtractor oftype HttpExtractor {
+    url: "https://gist.githubusercontent.com/noamross/e5d3e859aa0c794be10b/raw/b999fb4425b54c63cab088c0ce2c0d6ce961a563/cars.csv";
+  }
+
+
+  block CarsTextFileInterpreter oftype TextFileInterpreter { }
+
+  block CarsCSVInterpreter oftype CSVInterpreter {
+    enclosing: '"';
+  }
+
+  block NameHeaderWriter oftype CellWriter {
+    at: cell A1;
+
+    write: [
+      "name"
+    ];
+  }
+
+  block CarsTableInterpreter oftype TableInterpreter {
+    header: true;
+    columns: [
+      "name" oftype text,
+      "mpg" oftype decimal,
+      "cyl" oftype integer,
+      "disp" oftype decimal,
+      "hp" oftype integer,
+      "drat" oftype decimal,
+      "wt" oftype decimal,
+      "qsec" oftype decimal,
+      "vs" oftype integer,
+      "am" oftype integer,
+      "gear" oftype integer,
+      "carb" oftype integer
+    ];
+  }
+
+  transform IdentityTransform {
+    from mpg oftype decimal;
+    from cyl oftype integer;
+    to o oftype decimal;
+
+    o: mpg + cyl;
+  }
+
+  block MyTableTransformer oftype TableTransformer {
+    inputColumns: ['mpg', 'cyl'];
+    outputColumn: 'f(mpg, cyl)';
+
+    use: IdentityTransform;
+  }
+
+  block CarsLoader oftype SQLiteLoader {
+    table: "Cars";
+    file: "./cars.sqlite";
+  }
+}
+
diff --git a/libs/execution/src/lib/transforms/transform-executor.ts b/libs/execution/src/lib/transforms/transform-executor.ts
@@ -7,7 +7,6 @@ import { strict as assert } from 'assert';
 
 import {
   type EvaluationContext,
-  INTERNAL_VALUE_REPRESENTATION_TYPEGUARD,
   type InternalValueRepresentation,
   type PolarsInternal,
   type TransformDefinition,
@@ -89,7 +88,7 @@ export abstract class TransformExecutor<I, O> {
 
 export class PolarsTransformExecutor extends TransformExecutor<
   string[],
-  InternalValueRepresentation | PolarsInternal
+  PolarsInternal
 > {
   private static addInputColumnsToContext(
     inputDetailsList: readonly PortDetails[],
@@ -107,7 +106,7 @@ export class PolarsTransformExecutor extends TransformExecutor<
   protected override doExecuteTransform(
     inputColumns: string[],
     context: ExecutionContext,
-  ): InternalValueRepresentation | PolarsInternal | undefined {
+  ): PolarsInternal | undefined {
     const inputDetails = this.getInputDetails();
     const outputDetails = this.getOutputDetails();
 
@@ -117,10 +116,9 @@ export class PolarsTransformExecutor extends TransformExecutor<
       context.evaluationContext,
     );
 
-    let newValue: InternalValueRepresentation | PolarsInternal | undefined =
-      undefined;
+    let expr: PolarsInternal | undefined = undefined;
     try {
-      newValue = polarsEvaluateExpression(
+      expr = polarsEvaluateExpression(
         this.getOutputAssignment().expression,
         context.evaluationContext,
         context.wrapperFactories,
@@ -133,30 +131,15 @@ export class PolarsTransformExecutor extends TransformExecutor<
       }
     }
 
-    if (newValue === undefined) {
+    if (expr === undefined) {
       return undefined;
     }
 
-    if (INTERNAL_VALUE_REPRESENTATION_TYPEGUARD(newValue)) {
-      if (
-        !isValidValueRepresentation(newValue, outputDetails.valueType, context)
-      ) {
-        context.logger.logDebug(
-          `Invalid value: "${JSON.stringify(
-            newValue,
-          )}" does not match the type ${outputDetails.valueType.getName()}`,
-        );
-        return undefined;
-      }
-      return newValue;
-    }
-
-    // typeof newValue === PolarsInternal
     const otype = outputDetails.valueType.toPolarsDataType();
     if (otype === undefined) {
       return undefined;
     }
-    return newValue.cast(otype);
+    return expr.cast(otype);
   }
 }
 
@@ -257,7 +240,6 @@ export class TsTransformExecutor extends TransformExecutor<
 
   private addVariablesToContext(
     inputDetailsList: PortDetails[],
-
     columns: ReadonlyMap<string, TableColumn>,
     rowIndex: number,
     context: ExecutionContext,

diff --git a/libs/extensions/tabular/exec/src/lib/table-transformer-executor.ts b/libs/extensions/tabular/exec/src/lib/table-transformer-executor.ts
@@ -14,13 +14,7 @@ import {
   TsTransformExecutor,
   implementsStatic,
 } from '@jvalue/jayvee-execution';
-import {
-  INTERNAL_VALUE_REPRESENTATION_TYPEGUARD,
-  IOType,
-  type InternalValueRepresentation,
-  type PolarsInternal,
-} from '@jvalue/jayvee-language-server';
-import { pl } from 'nodejs-polars';
+import { IOType } from '@jvalue/jayvee-language-server';
 
 export abstract class TableTransformerExecutor extends AbstractBlockExecutor<
   IOType.TABLE,
@@ -119,16 +113,6 @@ export abstract class TableTransformerExecutor extends AbstractBlockExecutor<
 export class PolarsTableTransformerExecutor extends TableTransformerExecutor {
   public static readonly type = 'PolarsTableTransformer';
 
-  private newColumn(
-    x: InternalValueRepresentation | PolarsInternal,
-    nrows: number,
-  ): PolarsInternal | pl.Series {
-    if (INTERNAL_VALUE_REPRESENTATION_TYPEGUARD(x)) {
-      return pl.repeat(x, nrows);
-    }
-    return x;
-  }
-
   // eslint-disable-next-line @typescript-eslint/require-await
   override async doExecute(
     inputTable: R.PolarsTable,
@@ -172,7 +156,7 @@ export class PolarsTableTransformerExecutor extends TableTransformerExecutor {
       return checkInputColumnsMatchTransformInputTypesResult;
     }
 
-    const newValue = executor.executeTransform(inputColumnNames, context);
+    const expr = executor.executeTransform(inputColumnNames, context);
 
     this.logColumnOverwriteStatus(
       inputTable,
@@ -181,7 +165,7 @@ export class PolarsTableTransformerExecutor extends TableTransformerExecutor {
       executor.getOutputDetails(),
     );
 
-    if (newValue === undefined) {
+    if (expr === undefined) {
       return R.err({
         message: 'Skipping transform: Could not evaluate transform expression',
         diagnostic: {
@@ -190,8 +174,7 @@ export class PolarsTableTransformerExecutor extends TableTransformerExecutor {
       });
     }
 
-    const ncol = this.newColumn(newValue, inputTable.getNumberOfRows());
-    const ndf = inputTable.df.withColumn(ncol.alias(outputColumnName));
+    const ndf = inputTable.df.withColumn(expr.alias(outputColumnName));
     return R.ok(new R.PolarsTable(ndf));
   }
 }

diff --git a/libs/language-server/src/lib/ast/expressions/evaluate-expression.ts b/libs/language-server/src/lib/ast/expressions/evaluate-expression.ts
@@ -6,6 +6,7 @@
 import { strict as assert } from 'assert';
 
 import { assertUnreachable } from 'langium';
+import { pl } from 'nodejs-polars';
 
 import { type ValidationContext } from '../../validation';
 import {
@@ -37,8 +38,8 @@ import {
   type PolarsInternal,
 } from './internal-value-representation';
 import {
-  PolarsOperatorEvaluator,
   type OperatorEvaluator,
+  type PolarsOperatorEvaluator,
 } from './operator-evaluator';
 import {
   INTERNAL_VALUE_REPRESENTATION_TYPEGUARD,
@@ -111,7 +112,7 @@ function getEvaluator(
   if (isTernaryExpression(expression)) {
     const operator = expression.operator;
     throw new Error('Ternary operations are not supported yet');
-    //return evaluationContext.operatorRegistry.ternary[operator];
+    // return evaluationContext.operatorRegistry.ternary[operator];
   }
   assertUnreachable(expression);
 }
@@ -122,21 +123,29 @@ export function polarsEvaluateExpression(
   wrapperFactories: WrapperFactoryProvider,
   context: ValidationContext | undefined = undefined,
   strategy: EvaluationStrategy = EvaluationStrategy.LAZY,
-): InternalValueRepresentation | PolarsInternal | undefined {
+): PolarsInternal | undefined {
   if (expression === undefined) {
     return undefined;
   }
   if (isExpressionLiteral(expression)) {
     if (isFreeVariableLiteral(expression)) {
-      return evaluationContext.getValueFor(expression);
+      const fv = evaluationContext.getValueFor(expression);
+      if (INTERNAL_VALUE_REPRESENTATION_TYPEGUARD(fv)) {
+        return pl.lit(fv);
+      }
+      return fv;
     } else if (isValueLiteral(expression)) {
-      return evaluateValueLiteral(
+      const lit = evaluateValueLiteral(
         expression,
         evaluationContext,
         wrapperFactories,
         context,
         strategy,
       );
+      if (lit === undefined) {
+        return undefined;
+      }
+      return pl.lit(lit);
     }
     assertUnreachable(expression);
   }

diff --git a/libs/language-server/src/lib/ast/expressions/evaluators/addition-operator-evaluator.ts b/libs/language-server/src/lib/ast/expressions/evaluators/addition-operator-evaluator.ts
@@ -18,14 +18,9 @@ export class AdditionOperatorEvaluator extends DefaultBinaryOperatorEvaluator<
     return leftValue + rightValue;
   }
   override polarsDoEvaluate(
-    left: number | PolarsInternal,
-    right: number | PolarsInternal,
-  ): number | PolarsInternal {
-    if (NUMBER_TYPEGUARD(left)) {
-      return NUMBER_TYPEGUARD(right)
-        ? this.doEvaluate(left, right)
-        : right.plus(left);
-    }
+    left: PolarsInternal,
+    right: PolarsInternal,
+  ): PolarsInternal {
     return left.plus(right);
   }
 }
diff --git a/libs/language-server/src/lib/ast/expressions/evaluators/and-operator-evaluator.ts b/libs/language-server/src/lib/ast/expressions/evaluators/and-operator-evaluator.ts
@@ -4,7 +4,6 @@
 
 import { type PolarsInternal } from '../internal-value-representation';
 import { BooleanShortCircuitOperatorEvaluator } from '../operator-evaluator';
-import { BOOLEAN_TYPEGUARD } from '../typeguards';
 
 export class AndOperatorEvaluator extends BooleanShortCircuitOperatorEvaluator {
   constructor() {
@@ -22,15 +21,9 @@ export class AndOperatorEvaluator extends BooleanShortCircuitOperatorEvaluator {
     return leftValue && rightValue;
   }
   override polarsDoEvaluate(
-    leftValue: boolean | PolarsInternal,
-    rightValue: boolean | PolarsInternal,
-  ): boolean | PolarsInternal {
-    if (BOOLEAN_TYPEGUARD(leftValue)) {
-      if (BOOLEAN_TYPEGUARD(rightValue)) {
-        return this.polarsDoEvaluate(leftValue, rightValue);
-      }
-      return rightValue.and(leftValue);
-    }
+    leftValue: PolarsInternal,
+    rightValue: PolarsInternal,
+  ): PolarsInternal {
     return leftValue.and(rightValue);
   }
 }
diff --git a/libs/language-server/src/lib/ast/expressions/evaluators/division-operator-evaluator.ts b/libs/language-server/src/lib/ast/expressions/evaluators/division-operator-evaluator.ts
@@ -40,26 +40,9 @@ export class DivisionOperatorEvaluator extends DefaultBinaryOperatorEvaluator<
   }
 
   override polarsDoEvaluate(
-    left: number | PolarsInternal,
-    right: number | PolarsInternal,
-    expression: BinaryExpression,
-    context: ValidationContext | undefined,
-  ): number | PolarsInternal | undefined {
-    if (NUMBER_TYPEGUARD(left)) {
-      if (NUMBER_TYPEGUARD(right)) {
-        return this.doEvaluate(left, right, expression, context);
-      }
-      context?.accept(
-        'warning',
-        `<someNumber> / <someColumn> is not fully supported yet. Using a hack`,
-        {
-          node: expression,
-        },
-      );
-      // HACK:
-      const one = right.div(right);
-      return one.mul(left).div(right);
-    }
+    left: PolarsInternal,
+    right: PolarsInternal,
+  ): PolarsInternal {
     return left.div(right);
   }
 }
diff --git a/libs/language-server/src/lib/ast/expressions/evaluators/equality-operator-evaluator.ts b/libs/language-server/src/lib/ast/expressions/evaluators/equality-operator-evaluator.ts
@@ -28,14 +28,9 @@ export class EqualityOperatorEvaluator extends DefaultBinaryOperatorEvaluator<
     return left === right;
   }
   override polarsDoEvaluate(
-    left: InternalValueRepresentation | PolarsInternal,
-    right: InternalValueRepresentation | PolarsInternal,
-  ): boolean | PolarsInternal {
-    if (INTERNAL_VALUE_REPRESENTATION_TYPEGUARD(left)) {
-      return INTERNAL_VALUE_REPRESENTATION_TYPEGUARD(right)
-        ? this.doEvaluate(left, right)
-        : right.eq(left);
-    }
+    left: PolarsInternal,
+    right: PolarsInternal,
+  ): PolarsInternal {
     return left.eq(right);
   }
 }
diff --git a/libs/language-server/src/lib/ast/expressions/evaluators/greater-equal-operator-evaluator.ts b/libs/language-server/src/lib/ast/expressions/evaluators/greater-equal-operator-evaluator.ts
@@ -18,14 +18,9 @@ export class GreaterEqualOperatorEvaluator extends DefaultBinaryOperatorEvaluato
     return leftValue >= rightValue;
   }
   override polarsDoEvaluate(
-    left: number | PolarsInternal,
-    right: number | PolarsInternal,
-  ): boolean | PolarsInternal {
-    if (NUMBER_TYPEGUARD(left)) {
-      return NUMBER_TYPEGUARD(right)
-        ? this.doEvaluate(left, right)
-        : right.lt(left);
-    }
+    left: PolarsInternal,
+    right: PolarsInternal,
+  ): PolarsInternal {
     return left.gtEq(right);
   }
 }
diff --git a/libs/language-server/src/lib/ast/expressions/evaluators/greater-than-operator-evaluator.ts b/libs/language-server/src/lib/ast/expressions/evaluators/greater-than-operator-evaluator.ts
@@ -18,14 +18,9 @@ export class GreaterThanOperatorEvaluator extends DefaultBinaryOperatorEvaluator
     return leftValue > rightValue;
   }
   override polarsDoEvaluate(
-    left: number | PolarsInternal,
-    right: number | PolarsInternal,
-  ): boolean | PolarsInternal {
-    if (NUMBER_TYPEGUARD(left)) {
-      return NUMBER_TYPEGUARD(right)
-        ? this.doEvaluate(left, right)
-        : right.ltEq(left);
-    }
+    left: PolarsInternal,
+    right: PolarsInternal,
+  ): PolarsInternal {
     return left.gt(right);
   }
 }