diff --git a/sweep.yaml b/sweep.yaml
index e80c86cfbb..91a7fc2ebc 100644
--- a/sweep.yaml
+++ b/sweep.yaml
@@ -1,7 +1,7 @@
 gha_enabled: True
 branch: main
 blocked_dirs: [".github/"]
-draft: True
+draft: False
 description: "sweepai/sweep is a python 3.10 project. The main api endpoints are in sweepai/api.py. Write code that adheres to PEP8."
 # These are the docs that we index for Sweep.
 # We scrape these docs once a day and store them in our database.
diff --git a/sweepai/api.py b/sweepai/api.py
index 1791bffa3b..b55b82e0ba 100644
--- a/sweepai/api.py
+++ b/sweepai/api.py
@@ -441,6 +441,10 @@ async def webhook(raw_request: Request):
                 logger.info(f"Handling check suite for {request.repository.full_name}")
                 _, g = get_github_client(request.installation.id)
                 repo = g.get_repo(request.repository.full_name)
+                if request.check_run.pull_requests and issue.labels:
+                    issue = repo.get_issue(request.check_run.pull_requests[0].number)
+                    if not any(label.name == "sweep" for label in issue.labels):
+                        return {"success": True}
                 pull_request = repo.get_pull(request.check_run.pull_requests[0].number)
                 if (
                     len(request.check_run.pull_requests) > 0