From ed722d7740d38ed5340449456b7c442735b1ab9f Mon Sep 17 00:00:00 2001 From: Ameya Ketkar <94497232+ketkarameya@users.noreply.github.com> Date: Fri, 24 Feb 2023 08:17:29 -0800 Subject: [PATCH 1/2] Add tutorial for stale feature flag cleanup --- demo/stale_feature_flag_cleanup_demos.py | 82 ------ .../stale_feature_flag_cleanup_tutorial.ipynb | 262 ++++++++++++++++++ polyglot_piranha.pyi | 12 +- 3 files changed, 268 insertions(+), 88 deletions(-) delete mode 100644 demo/stale_feature_flag_cleanup_demos.py create mode 100644 demo/stale_feature_flag_cleanup_tutorial.ipynb diff --git a/demo/stale_feature_flag_cleanup_demos.py b/demo/stale_feature_flag_cleanup_demos.py deleted file mode 100644 index 42df76c1c..000000000 --- a/demo/stale_feature_flag_cleanup_demos.py +++ /dev/null @@ -1,82 +0,0 @@ -from os.path import join, dirname, getmtime, exists -from polyglot_piranha import execute_piranha, PiranhaArguments -import logging -from logging import info - -feature_flag_dir = join(dirname(__file__), "feature_flag_cleanup") - - -def run_java_ff_demo(): - info("Running the stale feature flag cleanup demo for Java") - - directory_path = join(feature_flag_dir, "java") - modified_file_path = join(directory_path, "SampleClass.java") - deleted_join_path = join(directory_path, "TestEnum.java") - configuration_path = join(directory_path, "configurations") - - old_mtime = getmtime(modified_file_path) - - args = PiranhaArguments( - "java", - { - "stale_flag_name": "SAMPLE_STALE_FLAG", - "treated": "true", - "treated_complement": "false", - }, - path_to_codebase=directory_path, - path_to_configurations=configuration_path, - ) - output_summary_java = execute_piranha(args) - - assert len(output_summary_java) == 2 - - for summary in output_summary_java: - assert len(summary.rewrites) > 0 - - new_mtime = getmtime(modified_file_path) - - assert old_mtime < new_mtime - assert not exists(deleted_join_path) - - -def run_kt_ff_demo(): - info("Running the stale feature flag cleanup demo for Kotlin") - - directory_path = join(feature_flag_dir, "kt") - modified_file_path = join(directory_path, "SampleClass.kt") - deleted_join_path = join(directory_path, "TestEnum.kt") - configuration_path = join(directory_path, "configurations") - - old_mtime = getmtime(modified_file_path) - - args = PiranhaArguments( - "kt", - { - "stale_flag_name": "SAMPLE_STALE_FLAG", - "treated": "true", - "treated_complement": "false", - }, - path_to_codebase=directory_path, - path_to_configurations=configuration_path, - ) - - output_summary_kt = execute_piranha(args) - - assert len(output_summary_kt) == 2 - - for summary in output_summary_kt: - assert len(summary.rewrites) > 0 - - new_mtime = getmtime(modified_file_path) - - assert old_mtime < new_mtime - assert not exists(deleted_join_path) - - -FORMAT = "%(levelname)s %(name)s %(asctime)-15s %(filename)s:%(lineno)d %(message)s" -logging.basicConfig(format=FORMAT) -logging.getLogger().setLevel(logging.DEBUG) - -run_java_ff_demo() -run_kt_ff_demo() -print("Completed running the stale feature flag cleanup demos") diff --git a/demo/stale_feature_flag_cleanup_tutorial.ipynb b/demo/stale_feature_flag_cleanup_tutorial.ipynb new file mode 100644 index 000000000..1210867a9 --- /dev/null +++ b/demo/stale_feature_flag_cleanup_tutorial.ipynb @@ -0,0 +1,262 @@ +{ + "cells": [ + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Piranha \n", + "\n", + "This is a guide for developing automation for ***cleaning up code related to stale feature flags*** \n", + "\n", + "Languages supported : Java, Kotlin, Go and Swift\n", + "\n", + "\n", + "ℹ️ **_NOTE:_** This tutorial is in Python for pedagogical purposes. \n", + "Piranha can be used with the CLI or using the Rust API. \n", + "We would encourage following this tutorial even if you do not intend to use the Python API.\n", + "\n", + "Now let's set up the notebook by installing the polyglot-piranha from source. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Please install [rust](https://rustup.rs/)\n", + "! python3 -m venv ../.env\n", + "! cd ../ && cargo build --no-default-features && maturin develop\n", + "\n", + "from polyglot_piranha import PiranhaArguments, execute_piranha, RuleGraph, Rule, OutgoingEdges" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": { + "tags": [] + }, + "source": [ + "
\n", + " CLI\n", + "

Build the executable from source

\n", + " \n", + " cargo build --no-default-features -r\n", + " \n", + "

The executable polyglot_piranha will be available under target/release

\n", + "
\n", + "\n", + "Now let's take the below example where we want to clean up the code related to the stale flag `SOME_STALE_FLAG` 🏴 because it has been permanently enabled." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "input_code = \"\"\"package com.uber.piranha;\n", + "class SampleJava {\n", + " \n", + " // An enum that declares the feature flag \n", + " enum FeatureFlag {\n", + " SOME_STALE_FLAG,\n", + " }\n", + "\n", + " public void sampleMethod(ExperimentInterface exp) {\n", + " if (exp.isToggleEnabled(SOME_STALE_FLAG)) {\n", + " System.out.println(\"SOME_STALE_FLAG is enabled\");\n", + " } else {\n", + " System.out.println(\"SOME_STALE_FLAG is disabled\");\n", + " }\n", + " }\n", + "}\n", + "\"\"\"" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "
\n", + " CLI\n", + "

A similar example for CLI is available here

\n", + " \n", + " cargo build --no-default-features -r\n", + " \n", + "

The executable polyglot_piranha will be available under target/release

\n", + "
\n", + "\n", + "\n", + "We will now craft a Piranha rule to update the feature flag API `exp.isToggleEnabled(SOME_STALE_FLAG)` and delete the enum case `SOME_STALE_FLAG`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# a rule to update `exp.isToggleEnabled(SOME_STALE_FLAG)`\n", + "update_rule_toggle_enabled = Rule (\n", + " name= \"update_toggle_enabled\",\n", + " query= \"\"\"((\n", + " (method_invocation \n", + " name : (_) @name\n", + " arguments: ((argument_list \n", + " ([\n", + " (field_access field: (_)@argument)\n", + " (_) @argument\n", + " ])) )\n", + " ) @method_invocation)\n", + " (#eq? @name \"isToggleEnabled\")\n", + " (#eq? @argument \"@stale_flag_name\")\n", + " )\"\"\",\n", + " replace_node=\"method_invocation\",\n", + " replace=\"@is_treated\",\n", + " holes= set([\"stale_flag_name\", \"is_treated\"]),\n", + " # Try commenting out the below line and see the difference in output\n", + " groups= set([\"replace_expression_with_boolean_literal\"])\n", + ")\n", + "\n", + "# a rule to delete enum case `SOME_STALE_FLAG`\n", + "delete_enum_case = Rule (\n", + " name = \"delete_enum_constant\",\n", + " query = \"\"\"(\n", + " ((enum_constant name : (_) @enum_case) @ec) \n", + " (#eq? @enum_case \"@stale_flag_name\")\n", + " )\"\"\",\n", + " replace_node = \"ec\",\n", + " replace = \"\",\n", + " holes = set([\"stale_flag_name\"]),\n", + " # Try commenting out the below line and see the difference in output\n", + " groups = set([\"delete_enum_entry\"]),\n", + ")\n", + "\n", + "rule_graph= RuleGraph(rules=[update_rule_toggle_enabled, delete_enum_case], edges=[])\n", + "\n", + "piranha_args = PiranhaArguments(\n", + " language = \"java\",\n", + " code_snippet= input_code,\n", + " rule_graph= rule_graph,\n", + " substitutions= {'stale_flag_name': 'SOME_STALE_FLAG', 'is_treated': 'true'},\n", + " dry_run= True,\n", + " cleanup_comments = True\n", + ")\n", + "output = execute_piranha(piranha_args)\n", + "output_content= output[0].content\n", + "assert output_content != input_code # sanity test for this tutorial\n", + "print(output_content)" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The rule `update_rule_toggle_enabled` specifies a rule that matches against an expression like `exp.isTreated(SOME_STALE_FLAG)` and replaces it with `true`. \n", + " \n", + "
\n", + " See the rule\n", + "\n", + " ##### `update_rule_toggle_enabled`\n", + " ```python Rule (\n", + " name= \"update_toggle_enabled\",\n", + " query= \"\"\"((\n", + " (method_invocation \n", + " name : (_) @name\n", + " arguments: ((argument_list \n", + " ([\n", + " (field_access field: (_)@argument)\n", + " (_) @argument\n", + " ])) )\n", + " ) @method_invocation)\n", + " (#eq? @name \"isToggleEnabled\")\n", + " (#eq? @argument \"@stale_flag_name\")\n", + " )\"\"\",\n", + " replace_node=\"method_invocation\",\n", + " replace=\"@is_treated\",\n", + " holes= set([\"stale_flag_name\", \"is_treated\"])\n", + " # Try un-commenting the below line and see the difference in output\n", + " groups= set([\"replace_expression_with_boolean_literal\"])\n", + ")\n", + " ```\n", + "
\n", + "\n", + "\n", + "The **`query`** property of the rule is a [tree-sitter query](https://tree-sitter.github.io/tree-sitter/using-parsers#pattern-matching-with-queries) that is matched against the source code to identify specific AST patterns.\n", + "\n", + "ℹ️ **_NOTE:_** Use [tree-sitter playground](https://tree-sitter.github.io/tree-sitter/playground) to craft the query that matches your specific APIs. This playground allows a user to create a query and match it against some input source code. This playground \"syntax highlights\" in the input code snippet based on input query. \n", + "
\n", + "Using the tree-sitter play ground locally \n", + "Install tree-sitter [CLI](https://github.com/tree-sitter/tree-sitter/blob/master/cli/README.md).\n", + "```bash\n", + "git clone https://github.com/tree-sitter/tree-sitter-java.git # Or any other tree-sitter language repo\n", + "cd tree-sitter-java\n", + "tree-sitter generate \n", + "tree-sitter build-wasm\n", + "tree-sitter playground\n", + "```\n", + "
\n", + "\n", + "The node captured by the tag-name specified in the **`replace_node`** property is replaced with the pattern specified in the **`replace`** property.\n", + "The **`replace`** pattern can use the tags from the **`query`** to construct a replacement based on the match (somewhat similar to [regex-replace](https://docs.microsoft.com/en-us/visualstudio/ide/using-regular-expressions-in-visual-studio?view=vs-2022)).\n", + "\n", + "Each rule also contains the **`groups`** property, that specifies the kind of change performed by this rule. Based on this group, appropriate\n", + "cleanup will be performed by Piranha. For instance, `replace_expression_with_boolean_literal` will trigger deep cleanups to eliminate dead code (like eliminating `consequent` of a `if statement`) caused by replacing an expression with a boolean literal.\n", + "Currently, Piranha provides deep clean-ups for edits that belong the groups - `replace_expression_with_boolean_literal`, `delete_enum_entry`, and `delete_method`. Adding an appropriate entry to the groups, hooks up the user defined rules to the pre-built cleanup rules.\n", + "\n", + "The property **`holes`** specify the holes or template variables `@treated` and `@stale_flag_name` that need to be replaced with some concrete values so that the rule matches only the feature flag API usages corresponding to a specific flag, and replace it specifically with `true` or `false`.\n", + "\n", + "The values for these template variables is specified via the `substitutions` dictionary. This dictionary is required to construct piranha arguments. \n", + "\n", + "
\n", + " CLI\n", + "

Java

\n", + " \n", + " ./target/release/polyglot_piranha -c demo/feature_flag_cleanup/java -f demo/feature_flag_cleanup/java/configurations -l \"java\" -s stale_flag_name=SAMPLE_STALE_FLAG -s treated=true -s treated_complement=false\n", + " \n", + " \n", + "

Kotlin

\n", + " \n", + " ./target/release/polyglot_piranha -c demo/feature_flag_cleanup/kt -f demo/feature_flag_cleanup/kt/configurations -l \"kt\" -s stale_flag_name=SAMPLE_STALE_FLAG -s treated=true -s treated_complement=false\n", + " \n", + "
" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.9" + }, + "vscode": { + "interpreter": { + "hash": "8940432f2365cfc2130f5c62b266313bc73e38258967dc33b60edac5998749e5" + } + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/polyglot_piranha.pyi b/polyglot_piranha.pyi index 0b043413d..c1bbd5990 100644 --- a/polyglot_piranha.pyi +++ b/polyglot_piranha.pyi @@ -38,12 +38,12 @@ class PiranhaArguments: rule_graph: Optional[RuleGraph]= None, path_to_codebase: Optional[str] = None, code_snippet: Optional[str] = None, - dry_run: Optional[bool] = None, - cleanup_comments: Optional[bool] = None, - cleanup_comments_buffer: Optional[int] = None, - number_of_ancestors_in_parent_scope: Optional[int] = None, - delete_file_if_empty : Optional[bool] = None, - delete_consecutive_new_lines : Optional[bool] = None, + dry_run: bool = False, + cleanup_comments: bool = False, + cleanup_comments_buffer: int = 2, + number_of_ancestors_in_parent_scope: int = None, + delete_file_if_empty : bool = True, + delete_consecutive_new_lines : bool = False, path_to_output: Optional[str] = None ): From 852bc29104eb1038901c2aea1d5a7b0c02f24357 Mon Sep 17 00:00:00 2001 From: Ameya Ketkar <94497232+ketkarameya@users.noreply.github.com> Date: Sat, 25 Feb 2023 08:38:45 -0800 Subject: [PATCH 2/2] . --- demo/custom_cleanups.ipynb | 53 +++++++++++++++++++ .../stale_feature_flag_cleanup_tutorial.ipynb | 2 +- 2 files changed, 54 insertions(+), 1 deletion(-) create mode 100644 demo/custom_cleanups.ipynb diff --git a/demo/custom_cleanups.ipynb b/demo/custom_cleanups.ipynb new file mode 100644 index 000000000..6de2f1789 --- /dev/null +++ b/demo/custom_cleanups.ipynb @@ -0,0 +1,53 @@ +{ + "cells": [ + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Custom Cleanups \n", + "\n", + "This is a guide for developing automation for ***custom code cleanups***.\n", + "\n", + "Languages supported : Java, Kotlin, Go, Python, Swift, and TypeScript\n", + "\n", + "ℹ️ **_NOTE:_** This tutorial is in Python for pedagogical purposes. \n", + "Piranha can be used with the CLI or using the Rust API. \n", + "We would encourage following this tutorial even if you do not intend to use the Python API.\n", + "\n", + "✅ **Prerequisite**: [Stale FF cleanup tutorial](./stale_feature_flag_cleanup_tutorial.ipynb)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": ".env", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.10.9" + }, + "orig_nbformat": 4, + "vscode": { + "interpreter": { + "hash": "8940432f2365cfc2130f5c62b266313bc73e38258967dc33b60edac5998749e5" + } + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/demo/stale_feature_flag_cleanup_tutorial.ipynb b/demo/stale_feature_flag_cleanup_tutorial.ipynb index 1210867a9..6d9e1237b 100644 --- a/demo/stale_feature_flag_cleanup_tutorial.ipynb +++ b/demo/stale_feature_flag_cleanup_tutorial.ipynb @@ -5,7 +5,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## Piranha \n", + "### Cleaning up code related to stale feature flags\n", "\n", "This is a guide for developing automation for ***cleaning up code related to stale feature flags*** \n", "\n",