diff --git a/.github/workflows/check-compatibility.yml b/.github/workflows/check-compatibility.yml
new file mode 100644
index 0000000000000..b208fe38a581f
--- /dev/null
+++ b/.github/workflows/check-compatibility.yml
@@ -0,0 +1,35 @@
+---
+name: Check Compatibility
+
+on:
+  pull_request_target
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Run compatibility task
+        run: ./gradlew checkCompatibility | tee $HOME/gradlew-check.out
+
+      - name: Get results
+        run: |
+          echo 'Compatibility status:' > ${{ github.workspace }}/results.txt && echo '```' >> ${{ github.workspace }}/results.txt
+          grep -e 'Compatible components' -e 'Incompatible components' -e 'Components skipped' -A 2 -B 3 $HOME/gradlew-check.out >> "${{ github.workspace }}/results.txt"
+          echo '```' >> ${{ github.workspace }}/results.txt
+
+      - name: GitHub App token
+        id: github_app_token
+        uses: tibdex/github-app-token@v1.6.0
+        with:
+          app_id: ${{ secrets.APP_ID }}
+          private_key: ${{ secrets.APP_PRIVATE_KEY }}
+          installation_id: 22958780
+
+      - name: Add comment on the PR
+        uses: peter-evans/create-or-update-comment@v3
+        with:
+          token: ${{ steps.github_app_token.outputs.token }}
+          issue-number: ${{ github.event.number }}
+          body-path: "${{ github.workspace }}/results.txt"
diff --git a/buildSrc/src/main/groovy/org/opensearch/gradle/CheckCompatibilityTask.groovy b/buildSrc/src/main/groovy/org/opensearch/gradle/CheckCompatibilityTask.groovy
index 81542bcb6d856..8d2ba51572def 100644
--- a/buildSrc/src/main/groovy/org/opensearch/gradle/CheckCompatibilityTask.groovy
+++ b/buildSrc/src/main/groovy/org/opensearch/gradle/CheckCompatibilityTask.groovy
@@ -40,8 +40,8 @@ class CheckCompatibilityTask extends DefaultTask {
 
     @TaskAction
     void checkCompatibility() {
-        logger.info("Checking compatibility for: $repositoryUrls for $ref")
         repositoryUrls.parallelStream().forEach { repositoryUrl ->
+            logger.lifecycle("Checking compatibility for: $repositoryUrl with ref: $ref")
             def tempDir = File.createTempDir()
             try {
                 if (cloneAndCheckout(repositoryUrl, tempDir)) {
@@ -81,8 +81,16 @@ class CheckCompatibilityTask extends DefaultTask {
 
     protected static List getRepoUrls() {
         def json = new JsonSlurper().parse(REPO_URL.toURL())
-        def labels = json.projects.values()
-        return labels as List
+        def repository = json.projects.values()
+        def repoUrls = replaceSshWithHttps(repository as List)
+        return repoUrls
+    }
+
+    protected static replaceSshWithHttps(List<String> repoList) {
+        repoList.replaceAll { element ->
+            element.replace("git@github.com:", "https://github.com/")
+        }
+        return repoList
     }
 
     protected boolean cloneAndCheckout(repoUrl, directory) {