4. Final_LLM_for_Evaluations.ipynb

{"cells":[{"cell_type":"markdown","metadata":{"id":"knphiOWgjdVU"},"source":["#Clinical Trials LLM"]},{"cell_type":"markdown","metadata":{"id":"QLUKlNKKyppp"},"source":["**Install necessary libraries**"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"collapsed":true,"executionInfo":{"elapsed":13476,"status":"ok","timestamp":1722555441428,"user":{"displayName":"Nainika Reddy","userId":"15989986900907940721"},"user_tz":420},"id":"QSoNUxmpyokA","outputId":"fe3eee84-58d0-4e26-c4f7-f9c3a7fb9992"},"outputs":[{"name":"stdout","output_type":"stream","text":["Collecting openai\n","  Downloading openai-1.37.1-py3-none-any.whl.metadata (22 kB)\n","Requirement already satisfied: anyio<5,>=3.5.0 in /usr/local/lib/python3.10/dist-packages (from openai) (3.7.1)\n","Requirement already satisfied: distro<2,>=1.7.0 in /usr/lib/python3/dist-packages (from openai) (1.7.0)\n","Collecting httpx<1,>=0.23.0 (from openai)\n","  Downloading httpx-0.27.0-py3-none-any.whl.metadata (7.2 kB)\n","Requirement already satisfied: pydantic<3,>=1.9.0 in /usr/local/lib/python3.10/dist-packages (from openai) (2.8.2)\n","Requirement already satisfied: sniffio in /usr/local/lib/python3.10/dist-packages (from openai) (1.3.1)\n","Requirement already satisfied: tqdm>4 in /usr/local/lib/python3.10/dist-packages (from openai) (4.66.4)\n","Requirement already satisfied: typing-extensions<5,>=4.7 in /usr/local/lib/python3.10/dist-packages (from openai) (4.12.2)\n","Requirement already satisfied: idna>=2.8 in /usr/local/lib/python3.10/dist-packages (from anyio<5,>=3.5.0->openai) (3.7)\n","Requirement already satisfied: exceptiongroup in /usr/local/lib/python3.10/dist-packages (from anyio<5,>=3.5.0->openai) (1.2.2)\n","Requirement already satisfied: certifi in /usr/local/lib/python3.10/dist-packages (from httpx<1,>=0.23.0->openai) (2024.7.4)\n","Collecting httpcore==1.* (from httpx<1,>=0.23.0->openai)\n","  Downloading httpcore-1.0.5-py3-none-any.whl.metadata (20 kB)\n","Collecting h11<0.15,>=0.13 (from httpcore==1.*->httpx<1,>=0.23.0->openai)\n","  Downloading h11-0.14.0-py3-none-any.whl.metadata (8.2 kB)\n","Requirement already satisfied: annotated-types>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from pydantic<3,>=1.9.0->openai) (0.7.0)\n","Requirement already satisfied: pydantic-core==2.20.1 in /usr/local/lib/python3.10/dist-packages (from pydantic<3,>=1.9.0->openai) (2.20.1)\n","Downloading openai-1.37.1-py3-none-any.whl (337 kB)\n","\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m337.0/337.0 kB\u001b[0m \u001b[31m7.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hDownloading httpx-0.27.0-py3-none-any.whl (75 kB)\n","\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m75.6/75.6 kB\u001b[0m \u001b[31m5.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hDownloading httpcore-1.0.5-py3-none-any.whl (77 kB)\n","\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m77.9/77.9 kB\u001b[0m \u001b[31m6.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hDownloading h11-0.14.0-py3-none-any.whl (58 kB)\n","\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m58.3/58.3 kB\u001b[0m \u001b[31m4.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hInstalling collected packages: h11, httpcore, httpx, openai\n","Successfully installed h11-0.14.0 httpcore-1.0.5 httpx-0.27.0 openai-1.37.1\n","Collecting langchain_openai\n","  Downloading langchain_openai-0.1.20-py3-none-any.whl.metadata (2.6 kB)\n","Collecting langchain-core<0.3.0,>=0.2.26 (from langchain_openai)\n","  Downloading langchain_core-0.2.26-py3-none-any.whl.metadata (6.2 kB)\n","Requirement already satisfied: openai<2.0.0,>=1.32.0 in /usr/local/lib/python3.10/dist-packages (from langchain_openai) (1.37.1)\n","Collecting tiktoken<1,>=0.7 (from langchain_openai)\n","  Downloading tiktoken-0.7.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (6.6 kB)\n","Requirement already satisfied: PyYAML>=5.3 in /usr/local/lib/python3.10/dist-packages (from langchain-core<0.3.0,>=0.2.26->langchain_openai) (6.0.1)\n","Collecting jsonpatch<2.0,>=1.33 (from langchain-core<0.3.0,>=0.2.26->langchain_openai)\n","  Downloading jsonpatch-1.33-py2.py3-none-any.whl.metadata (3.0 kB)\n","Collecting langsmith<0.2.0,>=0.1.75 (from langchain-core<0.3.0,>=0.2.26->langchain_openai)\n","  Downloading langsmith-0.1.95-py3-none-any.whl.metadata (13 kB)\n","Requirement already satisfied: packaging<25,>=23.2 in /usr/local/lib/python3.10/dist-packages (from langchain-core<0.3.0,>=0.2.26->langchain_openai) (24.1)\n","Requirement already satisfied: pydantic<3,>=1 in /usr/local/lib/python3.10/dist-packages (from langchain-core<0.3.0,>=0.2.26->langchain_openai) (2.8.2)\n","Collecting tenacity!=8.4.0,<9.0.0,>=8.1.0 (from langchain-core<0.3.0,>=0.2.26->langchain_openai)\n","  Downloading tenacity-8.5.0-py3-none-any.whl.metadata (1.2 kB)\n","Requirement already satisfied: typing-extensions>=4.7 in /usr/local/lib/python3.10/dist-packages (from langchain-core<0.3.0,>=0.2.26->langchain_openai) (4.12.2)\n","Requirement already satisfied: anyio<5,>=3.5.0 in /usr/local/lib/python3.10/dist-packages (from openai<2.0.0,>=1.32.0->langchain_openai) (3.7.1)\n","Requirement already satisfied: distro<2,>=1.7.0 in /usr/lib/python3/dist-packages (from openai<2.0.0,>=1.32.0->langchain_openai) (1.7.0)\n","Requirement already satisfied: httpx<1,>=0.23.0 in /usr/local/lib/python3.10/dist-packages (from openai<2.0.0,>=1.32.0->langchain_openai) (0.27.0)\n","Requirement already satisfied: sniffio in /usr/local/lib/python3.10/dist-packages (from openai<2.0.0,>=1.32.0->langchain_openai) (1.3.1)\n","Requirement already satisfied: tqdm>4 in /usr/local/lib/python3.10/dist-packages (from openai<2.0.0,>=1.32.0->langchain_openai) (4.66.4)\n","Requirement already satisfied: regex>=2022.1.18 in /usr/local/lib/python3.10/dist-packages (from tiktoken<1,>=0.7->langchain_openai) (2024.5.15)\n","Requirement already satisfied: requests>=2.26.0 in /usr/local/lib/python3.10/dist-packages (from tiktoken<1,>=0.7->langchain_openai) (2.31.0)\n","Requirement already satisfied: idna>=2.8 in /usr/local/lib/python3.10/dist-packages (from anyio<5,>=3.5.0->openai<2.0.0,>=1.32.0->langchain_openai) (3.7)\n","Requirement already satisfied: exceptiongroup in /usr/local/lib/python3.10/dist-packages (from anyio<5,>=3.5.0->openai<2.0.0,>=1.32.0->langchain_openai) (1.2.2)\n","Requirement already satisfied: certifi in /usr/local/lib/python3.10/dist-packages (from httpx<1,>=0.23.0->openai<2.0.0,>=1.32.0->langchain_openai) (2024.7.4)\n","Requirement already satisfied: httpcore==1.* in /usr/local/lib/python3.10/dist-packages (from httpx<1,>=0.23.0->openai<2.0.0,>=1.32.0->langchain_openai) (1.0.5)\n","Requirement already satisfied: h11<0.15,>=0.13 in /usr/local/lib/python3.10/dist-packages (from httpcore==1.*->httpx<1,>=0.23.0->openai<2.0.0,>=1.32.0->langchain_openai) (0.14.0)\n","Collecting jsonpointer>=1.9 (from jsonpatch<2.0,>=1.33->langchain-core<0.3.0,>=0.2.26->langchain_openai)\n","  Downloading jsonpointer-3.0.0-py2.py3-none-any.whl.metadata (2.3 kB)\n","Collecting orjson<4.0.0,>=3.9.14 (from langsmith<0.2.0,>=0.1.75->langchain-core<0.3.0,>=0.2.26->langchain_openai)\n","  Downloading orjson-3.10.6-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (50 kB)\n","\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m50.4/50.4 kB\u001b[0m \u001b[31m2.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hRequirement already satisfied: annotated-types>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from pydantic<3,>=1->langchain-core<0.3.0,>=0.2.26->langchain_openai) (0.7.0)\n","Requirement already satisfied: pydantic-core==2.20.1 in /usr/local/lib/python3.10/dist-packages (from pydantic<3,>=1->langchain-core<0.3.0,>=0.2.26->langchain_openai) (2.20.1)\n","Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests>=2.26.0->tiktoken<1,>=0.7->langchain_openai) (3.3.2)\n","Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests>=2.26.0->tiktoken<1,>=0.7->langchain_openai) (2.0.7)\n","Downloading langchain_openai-0.1.20-py3-none-any.whl (48 kB)\n","\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m48.2/48.2 kB\u001b[0m \u001b[31m3.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hDownloading langchain_core-0.2.26-py3-none-any.whl (378 kB)\n","\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m378.9/378.9 kB\u001b[0m \u001b[31m11.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hDownloading tiktoken-0.7.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.1 MB)\n","\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.1/1.1 MB\u001b[0m \u001b[31m33.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hDownloading jsonpatch-1.33-py2.py3-none-any.whl (12 kB)\n","Downloading langsmith-0.1.95-py3-none-any.whl (275 kB)\n","\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m275.8/275.8 kB\u001b[0m \u001b[31m16.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hDownloading tenacity-8.5.0-py3-none-any.whl (28 kB)\n","Downloading jsonpointer-3.0.0-py2.py3-none-any.whl (7.6 kB)\n","Downloading orjson-3.10.6-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (141 kB)\n","\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m141.1/141.1 kB\u001b[0m \u001b[31m11.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hInstalling collected packages: tenacity, orjson, jsonpointer, tiktoken, jsonpatch, langsmith, langchain-core, langchain_openai\n","  Attempting uninstall: tenacity\n","    Found existing installation: tenacity 9.0.0\n","    Uninstalling tenacity-9.0.0:\n","      Successfully uninstalled tenacity-9.0.0\n","Successfully installed jsonpatch-1.33 jsonpointer-3.0.0 langchain-core-0.2.26 langchain_openai-0.1.20 langsmith-0.1.95 orjson-3.10.6 tenacity-8.5.0 tiktoken-0.7.0\n"]}],"source":["!pip install openai --upgrade\n","!pip install langchain_openai --upgrade"]},{"cell_type":"markdown","metadata":{"id":"31hWI9orzDCm"},"source":["### Import and Pre-Process Clinical Trial Data from clinicaltrials.gov"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"collapsed":true,"executionInfo":{"elapsed":151882,"status":"ok","timestamp":1722555593302,"user":{"displayName":"Nainika Reddy","userId":"15989986900907940721"},"user_tz":420},"id":"Hc9Us193zBhU","outputId":"e190ccc3-1275-4a79-f7df-568b5887ff7a"},"outputs":[{"name":"stdout","output_type":"stream","text":["Fetching data from: https://clinicaltrials.gov/api/v2/studies?query.titles=Breast Cancer&pageSize=100\n","Fetching data from: https://clinicaltrials.gov/api/v2/studies?query.titles=Breast Cancer&pageSize=100&pageToken=NF0g5JCFl_Ms\n","Fetching data from: https://clinicaltrials.gov/api/v2/studies?query.titles=Breast Cancer&pageSize=100&pageToken=NF0g5JWCk_go\n","Fetching data from: https://clinicaltrials.gov/api/v2/studies?query.titles=Breast Cancer&pageSize=100&pageToken=NF0g5JqClvgr\n","Fetching data from: https://clinicaltrials.gov/api/v2/studies?query.titles=Breast Cancer&pageSize=100&pageToken=NF0g5JKFlPIhww\n","Fetching data from: https://clinicaltrials.gov/api/v2/studies?query.titles=Breast Cancer&pageSize=100&pageToken=NF0g5JKClfIhwA\n","Fetching data from: https://clinicaltrials.gov/api/v2/studies?query.titles=Breast Cancer&pageSize=100&pageToken=NF0g5JKPlPkqyA\n","Fetching data from: https://clinicaltrials.gov/api/v2/studies?query.titles=Breast Cancer&pageSize=100&pageToken=NF0g5JGFlvgrxA\n","Fetching data from: https://clinicaltrials.gov/api/v2/studies?query.titles=Breast Cancer&pageSize=100&pageToken=NF0g5JGCm_YuxA\n","Fetching data from: https://clinicaltrials.gov/api/v2/studies?query.titles=Breast Cancer&pageSize=100&pageToken=NF0g5JGOlPMvxg\n","Fetching data from: https://clinicaltrials.gov/api/v2/studies?query.titles=Breast Cancer&pageSize=100&pageToken=NF0g5JCEkfAgwg\n","Fetching data from: https://clinicaltrials.gov/api/v2/studies?query.titles=Breast Cancer&pageSize=100&pageToken=NF0g5JCBmvcqwQ\n","Fetching data from: https://clinicaltrials.gov/api/v2/studies?query.titles=Breast Cancer&pageSize=100&pageToken=NF0g5JeGkvIpxQ\n","Fetching data from: https://clinicaltrials.gov/api/v2/studies?query.titles=Breast Cancer&pageSize=100&pageToken=NF0g5JeDk_Mhxg\n","Fetching data from: https://clinicaltrials.gov/api/v2/studies?query.titles=Breast Cancer&pageSize=100&pageToken=NF0g5JeAm_AoxQ\n","Fetching data from: https://clinicaltrials.gov/api/v2/studies?query.titles=Breast Cancer&pageSize=100&pageToken=NF0g5JaGkvEvxA\n","Fetching data from: https://clinicaltrials.gov/api/v2/studies?query.titles=Breast Cancer&pageSize=100&pageToken=NF0g5JaCkvMvwg\n","Fetching data from: https://clinicaltrials.gov/api/v2/studies?query.titles=Breast Cancer&pageSize=100&pageToken=NF0g5JaPkvEuxQ\n","Fetching data from: https://clinicaltrials.gov/api/v2/studies?query.titles=Breast Cancer&pageSize=100&pageToken=NF0g5JWGkfUtyQ\n","Fetching data from: https://clinicaltrials.gov/api/v2/studies?query.titles=Breast Cancer&pageSize=100&pageToken=NF0g5JWDmvgtyQ\n","Fetching data from: https://clinicaltrials.gov/api/v2/studies?query.titles=Breast Cancer&pageSize=100&pageToken=NF0g5JWPkPYsyQ\n","Fetching data from: https://clinicaltrials.gov/api/v2/studies?query.titles=Breast Cancer&pageSize=100&pageToken=NF0g5JSFk_Igwg\n","Fetching data from: https://clinicaltrials.gov/api/v2/studies?query.titles=Breast Cancer&pageSize=100&pageToken=NF0g5JSCkfchwg\n","Fetching data from: https://clinicaltrials.gov/api/v2/studies?query.titles=Breast Cancer&pageSize=100&pageToken=NF0g5JSPkPkgxQ\n","Fetching data from: https://clinicaltrials.gov/api/v2/studies?query.titles=Breast Cancer&pageSize=100&pageToken=NF0g5JuGlPkoxw\n","Fetching data from: https://clinicaltrials.gov/api/v2/studies?query.titles=Breast Cancer&pageSize=100&pageToken=NF0g5JuCl_MgxQ\n","Fetching data from: https://clinicaltrials.gov/api/v2/studies?query.titles=Breast Cancer&pageSize=100&pageToken=NF0g5JuPkfQtwg\n","Fetching data from: https://clinicaltrials.gov/api/v2/studies?query.titles=Breast Cancer&pageSize=100&pageToken=NF0g5JqFkPMrwA\n","Fetching data from: https://clinicaltrials.gov/api/v2/studies?query.titles=Breast Cancer&pageSize=100&pageToken=NF0g5JqCkfcgww\n","Fetching data from: https://clinicaltrials.gov/api/v2/studies?query.titles=Breast Cancer&pageSize=100&pageToken=NF0g5JqOkPMqww\n","Fetching data from: https://clinicaltrials.gov/api/v2/studies?query.titles=Breast Cancer&pageSize=100&pageToken=NF0g5JKHkPAqxAI\n","Fetching data from: https://clinicaltrials.gov/api/v2/studies?query.titles=Breast Cancer&pageSize=100&pageToken=NF0g5JKHl_IvwwE\n","Fetching data from: https://clinicaltrials.gov/api/v2/studies?query.titles=Breast Cancer&pageSize=100&pageToken=NF0g5JKHmvMhyQc\n","Fetching data from: https://clinicaltrials.gov/api/v2/studies?query.titles=Breast Cancer&pageSize=100&pageToken=NF0g5JKGk_MqwgI\n","Fetching data from: https://clinicaltrials.gov/api/v2/studies?query.titles=Breast Cancer&pageSize=100&pageToken=NF0g5JKGlvQgxQY\n","Fetching data from: https://clinicaltrials.gov/api/v2/studies?query.titles=Breast Cancer&pageSize=100&pageToken=NF0g5JKGmvAvwg4\n","Fetching data from: https://clinicaltrials.gov/api/v2/studies?query.titles=Breast Cancer&pageSize=100&pageToken=NF0g5JKFk_UrwAQ\n","Fetching data from: https://clinicaltrials.gov/api/v2/studies?query.titles=Breast Cancer&pageSize=100&pageToken=NF0g5JKFl_ErxAE\n","Fetching data from: https://clinicaltrials.gov/api/v2/studies?query.titles=Breast Cancer&pageSize=100&pageToken=NF0g5JKFmvMpwg4\n","Fetching data from: https://clinicaltrials.gov/api/v2/studies?query.titles=Breast Cancer&pageSize=100&pageToken=NF0g5JKEk_YswQQ\n","Fetching data from: https://clinicaltrials.gov/api/v2/studies?query.titles=Breast Cancer&pageSize=100&pageToken=NF0g5JKElvkvwAA\n","Fetching data from: https://clinicaltrials.gov/api/v2/studies?query.titles=Breast Cancer&pageSize=100&pageToken=NF0g5JKEmvcpxwI\n","Fetching data from: https://clinicaltrials.gov/api/v2/studies?query.titles=Breast Cancer&pageSize=100&pageToken=NF0g5JKDk_kvxQc\n","Fetching data from: https://clinicaltrials.gov/api/v2/studies?query.titles=Breast Cancer&pageSize=100&pageToken=NF0g5JKDl_MoxQ4\n","Fetching data from: https://clinicaltrials.gov/api/v2/studies?query.titles=Breast Cancer&pageSize=100&pageToken=NF0g5JKDmvktwAE\n","Fetching data from: https://clinicaltrials.gov/api/v2/studies?query.titles=Breast Cancer&pageSize=100&pageToken=NF0g5JKCkPMhyQ8\n","Fetching data from: https://clinicaltrials.gov/api/v2/studies?query.titles=Breast Cancer&pageSize=100&pageToken=NF0g5JKCl_MrxwA\n","Fetching data from: https://clinicaltrials.gov/api/v2/studies?query.titles=Breast Cancer&pageSize=100&pageToken=NF0g5JKCmvUtwQM\n","Fetching data from: https://clinicaltrials.gov/api/v2/studies?query.titles=Breast Cancer&pageSize=100&pageToken=NF0g5JKBk_MswA8\n","Fetching data from: https://clinicaltrials.gov/api/v2/studies?query.titles=Breast Cancer&pageSize=100&pageToken=NF0g5JKBlvkpxwM\n","Fetching data from: https://clinicaltrials.gov/api/v2/studies?query.titles=Breast Cancer&pageSize=100&pageToken=NF0g5JKBmvEtwAQ\n","Fetching data from: https://clinicaltrials.gov/api/v2/studies?query.titles=Breast Cancer&pageSize=100&pageToken=NF0g5JKAk_YqwgE\n","Fetching data from: https://clinicaltrials.gov/api/v2/studies?query.titles=Breast Cancer&pageSize=100&pageToken=NF0g5JKAl_EoyAY\n","Fetching data from: https://clinicaltrials.gov/api/v2/studies?query.titles=Breast Cancer&pageSize=100&pageToken=NF0g5JKAmvAhwgI\n","Fetching data from: https://clinicaltrials.gov/api/v2/studies?query.titles=Breast Cancer&pageSize=100&pageToken=NF0g5JKPk_cpxQM\n","Fetching data from: https://clinicaltrials.gov/api/v2/studies?query.titles=Breast Cancer&pageSize=100&pageToken=NF0g5JKPlvUgwwM\n","Fetching data from: https://clinicaltrials.gov/api/v2/studies?query.titles=Breast Cancer&pageSize=100&pageToken=NF0g5JKPmvEuxQE\n","Fetching data from: https://clinicaltrials.gov/api/v2/studies?query.titles=Breast Cancer&pageSize=100&pageToken=NF0g5JKOkvgqxwc\n","Fetching data from: https://clinicaltrials.gov/api/v2/studies?query.titles=Breast Cancer&pageSize=100&pageToken=NF0g5JKOlvIswgM\n","Fetching data from: https://clinicaltrials.gov/api/v2/studies?query.titles=Breast Cancer&pageSize=100&pageToken=NF0g5JKOmvAtyAI\n","Fetching data from: https://clinicaltrials.gov/api/v2/studies?query.titles=Breast Cancer&pageSize=100&pageToken=NF0g5JGHkvghxg8\n","Fetching data from: https://clinicaltrials.gov/api/v2/studies?query.titles=Breast Cancer&pageSize=100&pageToken=NF0g5JGHl_ErwQU\n","Fetching data from: https://clinicaltrials.gov/api/v2/studies?query.titles=Breast Cancer&pageSize=100&pageToken=NF0g5JGHlfkgwAI\n","Fetching data from: https://clinicaltrials.gov/api/v2/studies?query.titles=Breast Cancer&pageSize=100&pageToken=NF0g5JGGk_EqyQA\n","Fetching data from: https://clinicaltrials.gov/api/v2/studies?query.titles=Breast Cancer&pageSize=100&pageToken=NF0g5JGGl_Evwg8\n","Fetching data from: https://clinicaltrials.gov/api/v2/studies?query.titles=Breast Cancer&pageSize=100&pageToken=NF0g5JGGlfcvwQI\n","Fetching data from: https://clinicaltrials.gov/api/v2/studies?query.titles=Breast Cancer&pageSize=100&pageToken=NF0g5JGFkvcowgA\n","Fetching data from: https://clinicaltrials.gov/api/v2/studies?query.titles=Breast Cancer&pageSize=100&pageToken=NF0g5JGFkfUpyAA\n","Fetching data from: https://clinicaltrials.gov/api/v2/studies?query.titles=Breast Cancer&pageSize=100&pageToken=NF0g5JGFlfUuwgI\n","Fetching data from: https://clinicaltrials.gov/api/v2/studies?query.titles=Breast Cancer&pageSize=100&pageToken=NF0g5JGEkvYoxwc\n","Fetching data from: https://clinicaltrials.gov/api/v2/studies?query.titles=Breast Cancer&pageSize=100&pageToken=NF0g5JGEkfcgyQE\n","Fetching data from: https://clinicaltrials.gov/api/v2/studies?query.titles=Breast Cancer&pageSize=100&pageToken=NF0g5JGElPkpxQY\n","Fetching data from: https://clinicaltrials.gov/api/v2/studies?query.titles=Breast Cancer&pageSize=100&pageToken=NF0g5JGDkvEvwwY\n","Fetching data from: https://clinicaltrials.gov/api/v2/studies?query.titles=Breast Cancer&pageSize=100&pageToken=NF0g5JGDkfQvwAc\n","Fetching data from: https://clinicaltrials.gov/api/v2/studies?query.titles=Breast Cancer&pageSize=100&pageToken=NF0g5JGDlPcvxwI\n","Fetching data from: https://clinicaltrials.gov/api/v2/studies?query.titles=Breast Cancer&pageSize=100&pageToken=NF0g5JGCkvQrxAA\n","Fetching data from: https://clinicaltrials.gov/api/v2/studies?query.titles=Breast Cancer&pageSize=100&pageToken=NF0g5JGCkfkvyQc\n","Fetching data from: https://clinicaltrials.gov/api/v2/studies?query.titles=Breast Cancer&pageSize=100&pageToken=NF0g5JGClfEvwQE\n","Fetching data from: https://clinicaltrials.gov/api/v2/studies?query.titles=Breast Cancer&pageSize=100&pageToken=NF0g5JGBkvchwwA\n","Fetching data from: https://clinicaltrials.gov/api/v2/studies?query.titles=Breast Cancer&pageSize=100&pageToken=NF0g5JGBkfYgxgA\n","Fetching data from: https://clinicaltrials.gov/api/v2/studies?query.titles=Breast Cancer&pageSize=100&pageToken=NF0g5JGBlPYpxQQ\n","Fetching data from: https://clinicaltrials.gov/api/v2/studies?query.titles=Breast Cancer&pageSize=100&pageToken=NF0g5JGBm_grxgM\n","Fetching data from: https://clinicaltrials.gov/api/v2/studies?query.titles=Breast Cancer&pageSize=100&pageToken=NF0g5JGAkfQtwA4\n","Fetching data from: https://clinicaltrials.gov/api/v2/studies?query.titles=Breast Cancer&pageSize=100&pageToken=NF0g5JGAlPIqwQM\n","Fetching data from: https://clinicaltrials.gov/api/v2/studies?query.titles=Breast Cancer&pageSize=100&pageToken=NF0g5JGPkvUsxg4\n","Fetching data from: https://clinicaltrials.gov/api/v2/studies?query.titles=Breast Cancer&pageSize=100&pageToken=NF0g5JGPkfQpxgY\n","Fetching data from: https://clinicaltrials.gov/api/v2/studies?query.titles=Breast Cancer&pageSize=100&pageToken=NF0g5JGPlfQvyQU\n","Fetching data from: https://clinicaltrials.gov/api/v2/studies?query.titles=Breast Cancer&pageSize=100&pageToken=NF0g5JGOk_AhwAE\n","Fetching data from: https://clinicaltrials.gov/api/v2/studies?query.titles=Breast Cancer&pageSize=100&pageToken=NF0g5JGOl_AqxQI\n","Fetching data from: https://clinicaltrials.gov/api/v2/studies?query.titles=Breast Cancer&pageSize=100&pageToken=NF0g5JGOmvYpwwM\n","Fetching data from: https://clinicaltrials.gov/api/v2/studies?query.titles=Breast Cancer&pageSize=100&pageToken=NF0g5JCHk_kuxAA\n","Fetching data from: https://clinicaltrials.gov/api/v2/studies?query.titles=Breast Cancer&pageSize=100&pageToken=NF0g5JCHlvkuxQE\n","Fetching data from: https://clinicaltrials.gov/api/v2/studies?query.titles=Breast Cancer&pageSize=100&pageToken=NF0g5JCHmvIvwAY\n","Fetching data from: https://clinicaltrials.gov/api/v2/studies?query.titles=Breast Cancer&pageSize=100&pageToken=NF0g5JCGkPAoxwA\n","Fetching data from: https://clinicaltrials.gov/api/v2/studies?query.titles=Breast Cancer&pageSize=100&pageToken=NF0g5JCGl_QqyAA\n","Fetching data from: https://clinicaltrials.gov/api/v2/studies?query.titles=Breast Cancer&pageSize=100&pageToken=NF0g5JCGmvIvxQY\n","Fetching data from: https://clinicaltrials.gov/api/v2/studies?query.titles=Breast Cancer&pageSize=100&pageToken=NF0g5JCFkPYoxgM\n","Fetching data from: https://clinicaltrials.gov/api/v2/studies?query.titles=Breast Cancer&pageSize=100&pageToken=NF0g5JCFlPYswwU\n","Fetching data from: https://clinicaltrials.gov/api/v2/studies?query.titles=Breast Cancer&pageSize=100&pageToken=NF0g5JCEkvEpxg4\n","Fetching data from: https://clinicaltrials.gov/api/v2/studies?query.titles=Breast Cancer&pageSize=100&pageToken=NF0g5JCEkfMvyQM\n","Fetching data from: https://clinicaltrials.gov/api/v2/studies?query.titles=Breast Cancer&pageSize=100&pageToken=NF0g5JCEl_QuxQ8\n","Fetching data from: https://clinicaltrials.gov/api/v2/studies?query.titles=Breast Cancer&pageSize=100&pageToken=NF0g5JCEm_AuxAQ\n","Fetching data from: https://clinicaltrials.gov/api/v2/studies?query.titles=Breast Cancer&pageSize=100&pageToken=NF0g5JCDkPYoxgI\n","Fetching data from: https://clinicaltrials.gov/api/v2/studies?query.titles=Breast Cancer&pageSize=100&pageToken=NF0g5JCDlPchyQ4\n","Fetching data from: https://clinicaltrials.gov/api/v2/studies?query.titles=Breast Cancer&pageSize=100&pageToken=NF0g5JCDm_QryQ4\n","Fetching data from: https://clinicaltrials.gov/api/v2/studies?query.titles=Breast Cancer&pageSize=100&pageToken=NF0g5JCCkfEswwY\n","Fetching data from: https://clinicaltrials.gov/api/v2/studies?query.titles=Breast Cancer&pageSize=100&pageToken=NF0g5JCClfArwAY\n"]}],"source":["import requests\n","import pandas as pd\n","\n","# Initial URL for the first API call\n","base_url = \"https://clinicaltrials.gov/api/v2/studies\"\n","params = {\n","    \"query.titles\": \"Breast Cancer\",\n","    \"pageSize\": 100\n","}\n","\n","# Initialize an empty list to store the data\n","data_list = []\n","\n","# Loop until there is no nextPageToken\n","while True:\n","    # Print the current URL (for debugging purposes)\n","    print(\"Fetching data from:\", base_url + '?' + '&'.join([f\"{k}={v}\" for k, v in params.items()]))\n","\n","    # Send a GET request to the API\n","    response = requests.get(base_url, params=params)\n","\n","    # Check if the request was successful\n","    if response.status_code == 200:\n","        data = response.json()  # Parse JSON response\n","        studies = data.get('studies', [])  # Extract the list of studies\n","\n","        # Loop through each study and extract specific information\n","        for study in studies:\n","            # Safely access nested keys\n","            nctId = study['protocolSection']['identificationModule'].get('nctId', 'Unknown')\n","            overallStatus = study['protocolSection']['statusModule'].get('overallStatus', 'Unknown')\n","            startDate = study['protocolSection']['statusModule'].get('startDateStruct', {}).get('date', 'Unknown Date')\n","            conditions = ', '.join(study['protocolSection']['conditionsModule'].get('conditions', ['No conditions listed']))\n","            acronym = study['protocolSection']['identificationModule'].get('acronym', 'Unknown')\n","\n","            # Extract interventions safely\n","            interventions_list = study['protocolSection'].get('armsInterventionsModule', {}).get('interventions', [])\n","            interventions = ', '.join([intervention.get('name', 'No intervention name listed') for intervention in interventions_list]) if interventions_list else \"No interventions listed\"\n","\n","            # Extract locations safely\n","            locations_list = study['protocolSection'].get('contactsLocationsModule', {}).get('locations', [])\n","            locations = ', '.join([f\"{location.get('city', 'No City')} - {location.get('country', 'No Country')}\" for location in locations_list]) if locations_list else \"No locations listed\"\n","\n","            # Extract dates and phases\n","            primaryCompletionDate = study['protocolSection']['statusModule'].get('primaryCompletionDateStruct', {}).get('date', 'Unknown Date')\n","            studyFirstPostDate = study['protocolSection']['statusModule'].get('studyFirstPostDateStruct', {}).get('date', 'Unknown Date')\n","            lastUpdatePostDate = study['protocolSection']['statusModule'].get('lastUpdatePostDateStruct', {}).get('date', 'Unknown Date')\n","            studyType = study['protocolSection']['designModule'].get('studyType', 'Unknown')\n","            phases = ', '.join(study['protocolSection']['designModule'].get('phases', ['Not Available']))\n","\n","            # Extract eligibility criteria\n","            #eligibilities = ', '.join(study['protocolSection']['eligibilityModule'].get('eligibilityCriteria', ['No criteria listed']))\n","\n","            eligibility = study['protocolSection']['eligibilityModule'].get('eligibilityCriteria', 'Unknown')\n","\n","            #eligibilities_list = study['protocolSection'].get('eligibilityModule', {}).get('eligibilityCriteria', [])\n","            #eligibilities = ', '.join([eligiblity.get('name', 'No criteria listed') for eligiblity in eligibilities_list]) if eligibilities_list else \"No eligibility listed\"\n","\n","            # Append the data to the list as a dictionary\n","            data_list.append({\n","                \"NCT ID\": nctId,\n","                \"Acronym\": acronym,\n","                \"Overall Status\": overallStatus,\n","                \"Start Date\": startDate,\n","                \"Conditions\": conditions,\n","                \"Interventions\": interventions,\n","                \"Locations\": locations,\n","                \"Primary Completion Date\": primaryCompletionDate,\n","                \"Study First Post Date\": studyFirstPostDate,\n","                \"Last Update Post Date\": lastUpdatePostDate,\n","                \"Study Type\": studyType,\n","                \"Phases\": phases,\n","                \"Eligibility\": eligibility\n","            })\n","\n","        # Check for nextPageToken and update the params or break the loop\n","        nextPageToken = data.get('nextPageToken')\n","        if nextPageToken:\n","            params['pageToken'] = nextPageToken  # Set the pageToken for the next request\n","        else:\n","            break  # Exit the loop if no nextPageToken is present\n","    else:\n","        print(\"Failed to fetch data. Status code:\", response.status_code)\n","        break\n","\n","# Create a DataFrame from the list of dictionaries\n","clinical_trials_data = pd.DataFrame(data_list)\n","\n","# Print the DataFrame\n","#print(clinical_trial_df)\n","\n","# Optionally, save the DataFrame to a CSV file\n","clinical_trials_data .to_csv(\"clinical_trials_data.csv\", index=False)"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"lFuDNDGXzg0t"},"outputs":[],"source":["clinical_trials_data = clinical_trials_data[clinical_trials_data['Overall Status'] == 'RECRUITING']"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":20,"status":"ok","timestamp":1722555593304,"user":{"displayName":"Nainika Reddy","userId":"15989986900907940721"},"user_tz":420},"id":"CTAXgwbxz7gn","outputId":"b8912f5e-8de8-4764-eca4-d62c1a2b023d"},"outputs":[{"name":"stdout","output_type":"stream","text":["Number of Clinical Trials: 1756\n"]}],"source":["print('Number of Clinical Trials: ' + str(len(clinical_trials_data)))"]},{"cell_type":"markdown","metadata":{"id":"M6J3t94fzfwo"},"source":["### Import Patient Records"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":29738,"status":"ok","timestamp":1722555623025,"user":{"displayName":"Nainika Reddy","userId":"15989986900907940721"},"user_tz":420},"id":"OEAMU3pCkSPi","outputId":"4a7fbded-099d-4282-b5dd-f788e8e82278"},"outputs":[{"name":"stdout","output_type":"stream","text":["Mounted at /content/drive/\n"]}],"source":["from google.colab import drive\n","drive.mount('/content/drive/', force_remount=True)"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"WhBsfIczlS1-"},"outputs":[],"source":["#!ls /content/drive/MyDrive/synthea_dataset"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"qTAtcRbgcj2D"},"outputs":[],"source":["def flatten_dataframe(df):\n","    for column in df.columns:\n","        if df[column].apply(lambda x: isinstance(x, (list, dict))).any():\n","            # Normalize the nested column\n","            normalized_df = pd.json_normalize(df[column].explode().dropna().tolist(), sep='_')\n","            normalized_df.columns = [f\"{column}_{sub_col}\" for sub_col in normalized_df.columns]\n","            # Drop the original nested column and join the normalized columns\n","            df = df.drop(columns=[column]).join(normalized_df)\n","    return df"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"q7Z53J8cjiRY"},"outputs":[],"source":["patient_files = [\"/content/drive/MyDrive/synthea_dataset/Paulita78_Watsica258_2dcefbbe-9f7c-4253-85f8-e1f0e74d90c1.json\",\n","                 \"/content/drive/MyDrive/synthea_dataset/Grace552_Little434_4198b779-4200-467c-acc5-5819803189fe.json\",\n","                 \"/content/drive/MyDrive/synthea_dataset/Dominique369_Daniel959_0e197e39-92cc-4621-b60b-0e0544fa7c49.json\",\n","                 \"/content/drive/MyDrive/synthea_dataset/Jeri234_Koss676_a0437435-67d7-49f0-8a7d-a2344094a8f9.json\",\n","                 \"/content/drive/MyDrive/synthea_dataset/Luis923_Cremin516_e06ba0d6-5338-4630-be9c-c7a7817fc890.json\",\n","                 \"/content/drive/MyDrive/synthea_dataset/Ronnie7_Greenfelder433_fddf3bac-f14d-45a9-a0b0-690435ea799b.json\",\n","                 \"/content/drive/MyDrive/synthea_dataset/Tambra47_Lang846_ae6b9280-f98d-455b-87fa-cc576ab01d2d.json\",\n","                 \"/content/drive/MyDrive/synthea_dataset/Veronique514_Koepp521_874cbeae-3145-40a1-940a-b5fed8a60f99.json\",\n","                 \"/content/drive/MyDrive/synthea_dataset/Landon622_Beier427_f49b5d35-10da-4fb7-80af-7bebe7047d9f.json\",\n","                 \"/content/drive/MyDrive/synthea_dataset/Adrienne302_Zulauf375_1c83fb03-d139-4626-bb39-4a062c22b533.json\",\n","                 ]\n","patient_data = {}\n","patient_names = []"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":1347,"status":"ok","timestamp":1722555624338,"user":{"displayName":"Nainika Reddy","userId":"15989986900907940721"},"user_tz":420},"id":"0aRrQP-KTPKl","outputId":"d74a321c-8b31-4106-de51-4d0f8978fc86"},"outputs":[{"name":"stdout","output_type":"stream","text":["Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"]}],"source":["from google.colab import drive\n","drive.mount('/content/drive')"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":17285,"status":"ok","timestamp":1722555641614,"user":{"displayName":"Nainika Reddy","userId":"15989986900907940721"},"user_tz":420},"id":"V5QG9Ht_4S64","outputId":"b6916c04-7336-4ea1-dab8-ee38ece613cc"},"outputs":[{"name":"stdout","output_type":"stream","text":["Paulita78_Watsica258\n","Grace552_Little434\n","Dominique369_Daniel959\n","Jeri234_Koss676\n","Luis923_Cremin516\n","Ronnie7_Greenfelder433\n","Tambra47_Lang846\n","Veronique514_Koepp521\n","Landon622_Beier427\n","Adrienne302_Zulauf375\n"]}],"source":["import json\n","from collections import defaultdict\n","import pandas as pd\n","\n","for patient_file in patient_files:\n","  patient_name = patient_file.split('/')[-1].split('.')[0]\n","  patient_name = '_'.join(patient_name.split('_')[:2])\n","  print(patient_name)\n","  patient_names.append(patient_name)\n","\n","  patient = {}\n","  #Load patient file\n","  with open(patient_file) as f:\n","    data = json.load(f)\n","\n","  # Extract the entries\n","  entries = data.get('entry', [])\n","\n","  # Group by resourceType\n","  resource_dict = {}\n","  for entry in entries:\n","    resource = entry.get('resource', {})\n","    resource_type = resource.get('resourceType', 'Unknown')\n","    if resource_type not in resource_dict:\n","        resource_dict[resource_type] = []\n","    resource_dict[resource_type].append(resource)\n","\n","  for resource_type, resources in resource_dict.items():\n","    # Normalize the entire resource dictionary and separate nested tags\n","    df = pd.json_normalize(resources, sep='_')\n","\n","    # Flatten the nested columns\n","    df = flatten_dataframe(df)\n","\n","    # Filter out inactive patient information\n","    #if 'status' in df.columns:\n","      #print(df['status'].unique())\n","        #df = df[(df['status'] == 'active') | (df['status'] == 'current') | (df['status'] == 'final') | (df['status'] == 'finished') | (df['status'] == 'completed')]\n","\n","    if 'id' in df.columns:\n","        df = df.drop(columns=['id'])\n","\n","    columns_to_drop = [col for col in df.columns if any(substring in col for substring in ['reference', 'system', 'telecom', 'identifier', 'Identifier', 'uid', 'Address', 'address'])]\n","    df = df.drop(columns=columns_to_drop)\n","\n","    df = flatten_dataframe(df) #handle second layer of nesting\n","\n","    patient[resource_type] = df\n","\n","  patient_data[patient_name] = patient"]},{"cell_type":"markdown","metadata":{"id":"GvCqL92N9EMD"},"source":["### **Part 1:** Query LLM with Full Patient Record and Full Clinical Trial Data\n","\n","### **Part 2:** Prompt LLM to Create a Patient Summary from the Full Patient Record\n","\n","### **Part 3:** Query LLM with Patient Summary and Full Clinical Trial Data"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"qZZ1f4O90Fn4"},"outputs":[],"source":["llm_responses = {}"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"P-eCyerOvtxo"},"outputs":[],"source":["import os\n","from langchain_openai import ChatOpenAI\n","\n","os.environ[\"OPENAI_API_KEY\"] = 'sk-proj-bbcCqgJsIPOhjmiRfNdyT3BlbkFJuDf9IePweh97i2lYZkTl' # API Key from Jieming\n","\n","#model\n","model = \"gpt-4-turbo\" # most powerful gpt model\n","llm = ChatOpenAI(temperature=0.1, model=model)"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":663896,"status":"ok","timestamp":1722556307758,"user":{"displayName":"Nainika Reddy","userId":"15989986900907940721"},"user_tz":420},"id":"Ss07QqGRQEKG","outputId":"c26e1d9d-91bb-4634-dda7-2a05a40c608d"},"outputs":[{"name":"stdout","output_type":"stream","text":["Paulita78_Watsica258\n","Grace552_Little434\n","Dominique369_Daniel959\n","Jeri234_Koss676\n","Luis923_Cremin516\n","Ronnie7_Greenfelder433\n","Tambra47_Lang846\n","Veronique514_Koepp521\n","Landon622_Beier427\n","Adrienne302_Zulauf375\n"]}],"source":["from datetime import date\n","today_date = date.today()\n","\n","for patient in patient_names:\n","  print(patient)\n","\n","  responses = {}\n","\n","  # Trial 1: Query LLM with Full Patient Data and Full Clinical Trial Data\n","  patient_df = patient_data[patient]\n","\n","  query1 = f\"\"\"You are a doctor seeing a patient diagnosed with a type of breast cancer with the following medical history in the mCODE format. {patient_df}. Extract information about the patient's cancer prognosis and treatment. The patient's age can be inferred from today's data, {today_date} and the patient’s date of birth. Use this information and the clinical trial inclusion/exclusion criteria to identify up to 4 clinical trials that the patient is eligible for. The patient must meet all of the criteria listed for the clinical trial. Identify and report the NCT ID number and the clinical trial eligibility (inclusion and exclusion) criteria. For each eligibility criterion for the trial, explain why the patient is eligible. If the patient does not match the criteria for any of the trials, explain why. {clinical_trials_data}'\n","\"\"\"\n","\n","  response1 = llm.invoke(query1)\n","\n","  responses['Full+Full'] = response1.content\n","  responses['Full+Full_Metadata'] = response1.response_metadata\n","\n","  # Trial 2: Query LLM to Write a Patient Summary\n","  query2 = f\"\"\"You are a doctor seeing a patient diagnosed with a type of breast cancer with the following health record in the mCODE format. Write a detailed summary for this patient's cancer diagnosis and prognosis. Report the clinical stage type of the cancer diagnosis and other descriptive information about the cancer. Report the gender and the patient's age as of today's date, {today_date}. Report the patient's comorbidities and treatments. {patient_df}'\n","\"\"\"\n","\n","  response2 = llm.invoke(query2)\n","\n","  responses['Patient_Summary'] = response2.content\n","  responses['Patient_Summary_Metadata'] = response2.response_metadata\n","\n","  # Trial 3: Query LLM with Patient Summary and and Full Clinical Trial Data\n","  query3 = f\"\"\"You are a doctor seeing a patient diagnosed with a type of breast cancer with the following medical history. {response2.content}. Use this information and the clinical trial inclusion/exclusion criteria to identify up to 4 clinical trials that the patient is eligible for. The patient must meet all of the criteria listed for the clinical trial. Identify and report the NCT ID number and the clinical trial eligibility (inclusion and exclusion) criteria. For each eligibility criterion for the trial, explain why the patient is eligible. If the patient does not match the criteria for any of the trials, explain why. {clinical_trials_data}.’\n","\"\"\"\n","\n","  response3 = llm.invoke(query3)\n","\n","  responses['Summary+Full'] = response3.content\n","  responses['Summary+Full_Metadata'] = response3.response_metadata\n","\n","  llm_responses[patient] = responses"]},{"cell_type":"markdown","metadata":{"id":"IRDzBADITX_F"},"source":["### **Part 4:** Query RAG-Enabled LLM with Patient Summary and Text-Based Clinical Trial Data\n","\n","\n"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":680},"executionInfo":{"elapsed":434,"status":"ok","timestamp":1722556307807,"user":{"displayName":"Nainika Reddy","userId":"15989986900907940721"},"user_tz":420},"id":"eLCW5EMtQD_Y","outputId":"b66d013a-17ec-4d5e-a352-5de6add1e03a"},"outputs":[{"data":{"text/html":["<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>text</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>6</th>\n","      <td>The trial with NCT ID number NCT05939830 has the following eligbility criteria, for inclusion and exclusion Inclusion Criteria:\\n\\n1. Female aged between 18 and 70 years;\\n2. Pathologically confirmed invasive breast cancer (regardless of pathological type) with a clinical stage cT1-3;\\n3. Pathologically confirmed positive axillary lymph nodes with a clinical stage of N1-3;\\n4. Receiving a full course of neoadjuvant therapy (including neoadjuvant chemotherapy, neoadjuvant targeted therapy, neoadjuvant immunotherapy);\\n5. Positive axillary lymph nodes successfully stained by carbon nanoparticles injection with/without titanium clip marking;\\n6. All patients are required to undergo immunohistochemical staining for Estrogen Receptor (ER), Progesterone Receptor (PR), human epidermal growth factor receptor 2 (HER2), Ki- 67 proliferation index, and further fluorescence in situ hybridization (FISH) should be performed in HER2 2+ cases;\\n7. Preoperative clinical assessment (including physical examination, imaging, with or without nomogram assessment) suggests positive axillary lymph nodes converted to negative (ycN0);\\n8. ECOG score 0 - 1;\\n9. Patients voluntarily participated in this study and signed the informed consent form\\n\\nExclusion Criteria:\\n\\n1. Bilateral breast cancer;\\n2. Breast cancer during lactation period or pregnancy;\\n3. Physical examination or imaging examination confirmed presence of distant metastases;\\n4. Previous history of malignant tumor;\\n5. History of previous surgery on the affected axilla; or history of surgery affecting the function of the upper extremity;\\n6. History of radiation therapy to the breast or chest;\\n7. Positive incision margins for breast-conserving surgery/mastectomy;\\n8. Positive results of intraoperative rapid freeze pathology (including isolated tumor cells and micrometastases) for SrLNB (ypN+);\\n9. Those who unable to complete the full course of follow-up adjuvant therapy as prescribed for various reasons;\\n10. Aspartate transaminase (AST) and alanine transaminase (ALT) ≥ 1.5 times the upper limit of normal, alkaline phosphatase（ALP） ≥ 2.5 times the upper limit of normal, total bile ≥ 1.5 times the upper limit of normal, serum creatinine ≥ 1.5 times the upper limit of normal; Left Ventricular Ejection Fractions (LVEF) \\&lt; 50% in cardiac ultrasound;\\n11. Severe coagulation dysfunction, serious systemic disease, or uncontrolled infection;\\n12. Without personal freedom and independent civil capacity;\\n13. Those with mental disorders, addictions, who were not eligible for enrollment in the judgment of the investigator.</td>\n","    </tr>\n","    <tr>\n","      <th>8</th>\n","      <td>The trial with NCT ID number NCT03147430 has the following eligbility criteria, for inclusion and exclusion Inclusion Criteria:\\n\\n* Women who receive a suspicious mammogram report or are scheduled to receive testing for suspect breast area, with a subsequent biopsy to confirm diagnosis\\n* Willingness and ability to donate biospecimens for the purpose of propelling research.\\n* Participants aged ≥ 18.\\n\\nExclusion Criteria:\\n\\n* Individuals under 18 years of age or over 89 years of age.\\n* A known history of breast cancer.\\n* A diagnosis or history of any other type of cancer.\\n* Participants who are male.</td>\n","    </tr>\n","    <tr>\n","      <th>10</th>\n","      <td>The trial with NCT ID number NCT04638725 has the following eligbility criteria, for inclusion and exclusion For inclusion in the study, patients must be affiliated to the national or local social security, and must meet all the following criteria:\\n\\nInclusion Criteria:\\n\\n* Age ≥ 18 years\\n* Histological diagnosis of breast adenocarcinoma. Non-metastatic and operable.\\n* Current or prior treatment with one therapy targeting HER2 in adjuvant or neoadjuvant phase for the current breast cancer\\n* Given written informed consent\\n\\nExclusion Criteria:\\n\\n* Patients not able to comply to the protocol assessments for geographic, social or psychological reasons\\n* Patients placed under judicial protection, guardianship, or supervision\\n* History of cancer in the 5 years preceding anti-HER2 therapy initiation\\n* Concomitant cancer (except for an other non metastatic cancer treated only with surgery)\\n\\nNote : Patients are eligible at any time of the follow-up if the adjuvant or neoadjuvant chemotherapy started after 01/01/2019. Patients treated with trastuzumab, pertuzumab, neratinib or T-DM1 in a clinical trial are eligible in the SIGHER study.</td>\n","    </tr>\n","    <tr>\n","      <th>20</th>\n","      <td>The trial with NCT ID number NCT05989776 has the following eligbility criteria, for inclusion and exclusion Inclusion Criteria:\\n\\n* Patients over 18 and under 40 years of age treated for newly diagnosed breast cancer and receiving chemotherapy.\\n* People who are able to read the study information poster and the fertility preservation information brochure in French, either alone or with help from a caregiver or relative or an interpreter.\\n* People with intellectual disability will be included as long as they are able able to read the study information poster and the fertility preservation information brochure in French alone or with help from a caregiver or relative or an interpreter..\\n\\nExclusion Criteria:\\n\\n* People whose mental health status precludes participation in the study, as determined by the clinical team.</td>\n","    </tr>\n","    <tr>\n","      <th>21</th>\n","      <td>The trial with NCT ID number NCT04506476 has the following eligbility criteria, for inclusion and exclusion Inclusion Criteria:\\n\\n* Capacity for consent\\n* Minimum age 18\\n* Presence of breast cancer\\n* ECOG 0-2\\n* Indication for adjuvant radiotherapy of breast cancer after breast-conserving surgery or Ablatio mammae\\n\\nExclusion Criteria:\\n\\n* Participation in any other interventional study\\n* Pregnancy\\n* Contraindication against physical activity/sport and others\\n* Severe cardiovascular pre-existing conditions (after myocardial infarction, apoplexy in the last 6 months, congestive heart failure NYHA \\&gt; I°)\\n* preexisting diseases with are relevantly accompanied by a limited mobility in patients (e.g. paraparesis of the lower limbs)\\n* ECOG Status 3-4\\n* prior use of activity trackers</td>\n","    </tr>\n","  </tbody>\n","</table><br><label><b>dtype:</b> object</label>"],"text/plain":["6     The trial with NCT ID number NCT05939830 has t...\n","8     The trial with NCT ID number NCT03147430 has t...\n","10    The trial with NCT ID number NCT04638725 has t...\n","20    The trial with NCT ID number NCT05989776 has t...\n","21    The trial with NCT ID number NCT04506476 has t...\n","Name: text, dtype: object"]},"execution_count":14,"metadata":{},"output_type":"execute_result"}],"source":["from openai import OpenAI\n","\n","client = OpenAI(\n","  api_key=os.environ[\"OPENAI_API_KEY\"],  # this is also the default, it can be omitted\n",")\n","\n","clinical_trials_data['text'] = 'The trial with NCT ID number ' + clinical_trials_data['NCT ID'] + ' has the following eligbility criteria, for inclusion and exclusion ' + clinical_trials_data['Eligibility']\n","clinical_trials_data.head()['text']"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"R2HIGvloQDxl"},"outputs":[],"source":["import openai\n","import tiktoken\n","from scipy import spatial\n","import pandas as pd\n","import ast\n","import openai\n","\n","# Function to get text embeddings\n","def text_embedding(text):\n","    response = client.embeddings.create(model=\"text-embedding-ada-002\", input=text)\n","    return response.data[0].embedding\n","\n","# Apply the text_embedding function to the 'text' column\n","clinical_trials_data['embedding'] = clinical_trials_data['text'].apply(lambda x: text_embedding(x))\n","\n","#print(clinical_trials_data.head())"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"2BheDcmnSEDY"},"outputs":[],"source":["def strings_ranked_by_relatedness(\n","    query: str,\n","    df: pd.DataFrame,\n","    relatedness_fn = lambda x, y: 1 - spatial.distance.cosine(x, y),\n","    top_n: int = 1000\n","):\n","\n","    EMBEDDING_MODEL = \"text-embedding-ada-002\"\n","    query_embedding_response = openai.embeddings.create(\n","        model = EMBEDDING_MODEL,\n","        input = query,\n","    )\n","    query_embedding = query_embedding_response.data[0].embedding\n","    strings_and_relatednesses = [\n","        (row[\"text\"], relatedness_fn(query_embedding, row[\"embedding\"]))\n","        for i, row in df.iterrows()\n","    ]\n","    strings_and_relatednesses.sort(key = lambda x: x[1], reverse = True)\n","    strings, relatednesses = zip(*strings_and_relatednesses)\n","    return strings[:top_n], relatednesses[:top_n]\n","\n","def num_tokens(text: str) -> int:\n","    encoding = tiktoken.encoding_for_model(\"gpt-4o\")\n","    return len(encoding.encode(text))\n","\n","def query_message(\n","    query: str,\n","    df: pd.DataFrame,\n","    model: str,\n","    token_budget: int\n",") :\n","    strings, relatednesses = strings_ranked_by_relatedness(query, df)\n","    introduction = 'Use the below clinical trial information to match patients to clinical trials based on their eligibility criteria. Explain why the patient does or does not match the eligibility criteria.'\n","    question = f\"\\n\\nQuestion: {query}\"\n","    message = introduction\n","    for string in strings:\n","        next_row = f'\\n\\nClinical trial section:\\n\"\"\"\\n{string}\\n\"\"\"'\n","        if (\n","            num_tokens(message + next_row + question)\n","            > token_budget\n","        ):\n","            break\n","        else:\n","            message += next_row\n","    return message + question\n","\n","def ask(\n","    query: str,\n","    df: pd.DataFrame = clinical_trials_data,\n","    model: str = \"gpt-4o\",\n","    token_budget: int = 25000,\n","    print_message: bool = False,\n",") :\n","    message = query_message(query, df, model=model, token_budget = token_budget)\n","    if print_message:\n","        print(message)\n","    messages = [\n","        {\"role\": \"system\", \"content\": \"You match patients to clinical trials that they are eligible for.\"},\n","        {\"role\": \"user\", \"content\": message},\n","    ]\n","    response = openai.chat.completions.create(\n","        model = model,\n","        messages = messages,\n","        temperature = 0\n","    )\n","\n","    return response"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"background_save":true},"id":"PtyDaqTcSD5z"},"outputs":[],"source":["for patient in patient_names:\n","\n","  patient_summary = llm_responses[patient]['Patient_Summary']\n","\n","  # Trial 4: Query LLM to Write a Patient Summary\n","  query4 = query = f\"\"\"You are a doctor seeing a patient diagnosed with a type of breast cancer with the following medical history. {patient_summary}. Use this information and the clinical trial inclusion/exclusion criteria to identify up to 4 clinical trials that the patient is eligible for. The patient must meet all of the criteria listed for the clinical trial. Identify and report the NCT ID number and the clinical trial eligibility (inclusion and exclusion) criteria. For each eligibility criterion for the trial, explain why the patient is eligible. If the patient does not match the criteria for any of the trials, explain why.\"\"\"\n","\n","  response4 = ask(query4)\n","\n","  llm_responses[patient]['Summary+RAG'] = response4.choices[0].message.content\n","  llm_responses[patient]['Summary+RAG_Metadata'] = response4"]},{"cell_type":"markdown","metadata":{"id":"wJCO5Zj_57s0"},"source":["### **Part 5:** Query RAG-Enabled LLM with Patient Summary and Text-Based Clinical Trial Data Using an Alternative Relevancy Metric\n","\n","The model now calculates the patient summary's relatedness with the inclusion criteria and the unrelatedness with the exclusion criteria."]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"background_save":true},"id":"47IBvZnn7IUd"},"outputs":[],"source":["clinical_trials_data2 = clinical_trials_data.copy()\n","clinical_trials_data2 = clinical_trials_data2.drop(['embedding'], axis=1)"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"background_save":true},"id":"EDjEwYCcTiDQ"},"outputs":[],"source":["def split_eligibility(row):\n","    parts = row['Eligibility'].split('Exclusion Criteria:')\n","    if len(parts) == 2:\n","        inclusion_criteria = parts[0].strip()\n","        exclusion_criteria = parts[1].strip()\n","    else:\n","        inclusion_criteria = row['Eligibility'].strip()\n","        exclusion_criteria = ''\n","    return inclusion_criteria, exclusion_criteria\n","\n","# Apply the function to each row\n","clinical_trials_data2[['Inclusion_Criteria', 'Exclusion_Criteria']] = clinical_trials_data2.apply(split_eligibility, axis=1, result_type='expand')\n","\n","# Create the new columns with the formatted strings\n","clinical_trials_data2['text_inclusion'] = 'The trial with NCT ID number ' + clinical_trials_data2['NCT ID'] + ' has the following eligibility inclusion criteria: ' + clinical_trials_data2['Inclusion_Criteria']\n","clinical_trials_data2['text_exclusion'] = 'The trial with NCT ID number ' + clinical_trials_data2['NCT ID'] + ' has the following eligibility exclusion criteria: Exclusion Criteria: ' + clinical_trials_data2['Exclusion_Criteria']\n","\n","# Display the head of the new columns\n","#print(clinical_trials_data2.head()[['text_inclusion', 'text_exclusion']])"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"background_save":true},"id":"9qlOftC_TiJz"},"outputs":[],"source":["# Apply the text_embedding function to the 'text' column\n","clinical_trials_data2['embedding_inclusion'] = clinical_trials_data2['text_inclusion'].apply(lambda x: text_embedding(x))\n","clinical_trials_data2['embedding_exclusion'] = clinical_trials_data2['text_exclusion'].apply(lambda x: text_embedding(x))\n","\n","#print(clinical_trials_data2.head())"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"background_save":true},"id":"Tt6AA5tETiQ-"},"outputs":[],"source":["# Function to compute the combined score\n","def combined_score(query_embedding, row, relatedness_fn, unrelatedness_fn):\n","    relatedness = relatedness_fn(query_embedding, row[\"embedding_inclusion\"])\n","    unrelatedness = unrelatedness_fn(query_embedding, row[\"embedding_exclusion\"])\n","    return relatedness - unrelatedness  # Higher relatedness and lower unrelatedness\n","\n","# Function to rank strings by combined score\n","def strings_ranked_by_combined_score(\n","    query: str,\n","    df: pd.DataFrame,\n","    relatedness_fn = lambda x, y: 1 - spatial.distance.cosine(x, y),\n","    unrelatedness_fn = lambda x, y: spatial.distance.cosine(x, y),\n","    top_n: int = 1000\n","):\n","    EMBEDDING_MODEL = \"text-embedding-ada-002\"\n","    query_embedding_response = openai.embeddings.create(\n","        model = EMBEDDING_MODEL,\n","        input = query,\n","    )\n","    query_embedding = query_embedding_response.data[0].embedding\n","    strings_and_scores = [\n","        (row[\"text\"], combined_score(query_embedding, row, relatedness_fn, unrelatedness_fn))\n","        for i, row in df.iterrows()\n","    ]\n","    strings_and_scores.sort(key=lambda x: x[1], reverse=True)\n","    strings, scores = zip(*strings_and_scores)\n","    return strings[:top_n], scores[:top_n]\n","\n","\n","# Function to generate query message\n","def query_message2(\n","    query: str,\n","    df: pd.DataFrame,\n","    model: str,\n","    token_budget: int\n","):\n","    strings, scores = strings_ranked_by_combined_score(query, df)\n","    introduction = 'Use the below clinical trial information to match patients to clinical trials based on their eligibility criteria. Explain why the patient does or does not match the eligibility criteria.'\n","    question = f\"\\n\\nQuestion: {query}\"\n","    message = introduction\n","    for string in strings:\n","        next_row = f'\\n\\nClinical trial section:\\n\"\"\"\\n{string}\\n\"\"\"'\n","        if num_tokens(message + next_row + question) > token_budget:\n","            break\n","        else:\n","            message += next_row\n","    return message + question\n","\n","# Function to ask the model\n","def ask2(\n","    query: str,\n","    df: pd.DataFrame = clinical_trials_data2,\n","    model: str = \"gpt-4o\",\n","    token_budget: int = 25000,\n","    print_message: bool = False,\n","):\n","    message = query_message2(query, df, model=model, token_budget=token_budget)\n","    if print_message:\n","        print(message)\n","    messages = [\n","        {\"role\": \"system\", \"content\": \"You match patients to clinical trials that they are eligible for.\"},\n","        {\"role\": \"user\", \"content\": message},\n","    ]\n","    response = openai.chat.completions.create(\n","        model=model,\n","        messages=messages,\n","        temperature=0\n","    )\n","\n","    return response"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"background_save":true},"id":"L8tcSs1CTiVQ"},"outputs":[],"source":["for patient in patient_names:\n","\n","  patient_summary = llm_responses[patient]['Patient_Summary']\n","\n","  # Trial 5: Query LLM with Alternate Relatedness Calculation\n","  query5 = f\"\"\"You are a doctor seeing a patient diagnosed with a type of breast cancer with the following medical history. {patient_summary}. Use this information and the clinical trial inclusion/exclusion criteria to identify up to 4 clinical trials that the patient is eligible for. The patient must meet all of the criteria listed for the clinical trial. Identify and report the NCT ID number and the clinical trial eligibility (inclusion and exclusion) criteria. For each eligibility criterion for the trial, explain why the patient is eligible. If the patient does not match the criteria for any of the trials, explain why.\"\"\"\n","\n","  response5 = ask2(query5)\n","\n","  llm_responses[patient]['Summary+AlternateRAG'] = response5.choices[0].message.content\n","  llm_responses[patient]['Summary+AlternateRAG_Metadata'] = response5"]},{"cell_type":"markdown","metadata":{"id":"BFTifGe0Ug-y"},"source":["### **Save Files**"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"background_save":true},"id":"EAvrzz4mSDvN"},"outputs":[],"source":["llm_responses_df = pd.DataFrame.from_dict(llm_responses, orient='index')\n","llm_responses_df.to_csv('llm_responses.csv')"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"background_save":true},"id":"ioOdj9MjFTT2","outputId":"bdcd9b02-3bfa-4dc5-e5bb-22c55ae3b92b"},"outputs":[{"name":"stdout","output_type":"stream","text":["                                                                Full+Full  \\\n","Paulita78_Watsica258    Based on the provided data, the patient is a f...   \n","Grace552_Little434      To determine the eligibility of the patient fo...   \n","Dominique369_Daniel959  To determine the patient's eligibility for cli...   \n","Jeri234_Koss676         To determine the patient's eligibility for cli...   \n","Luis923_Cremin516       To determine the patient's eligibility for cli...   \n","Ronnie7_Greenfelder433  Based on the provided mCODE data for the patie...   \n","Tambra47_Lang846        Based on the provided data and the patient's m...   \n","Veronique514_Koepp521   To determine the patient's eligibility for cli...   \n","Landon622_Beier427      To determine the patient's eligibility for cli...   \n","Adrienne302_Zulauf375   To determine the patient's eligibility for cli...   \n","\n","                                                       Full+Full_Metadata  \\\n","Paulita78_Watsica258    {'token_usage': {'completion_tokens': 539, 'pr...   \n","Grace552_Little434      {'token_usage': {'completion_tokens': 598, 'pr...   \n","Dominique369_Daniel959  {'token_usage': {'completion_tokens': 655, 'pr...   \n","Jeri234_Koss676         {'token_usage': {'completion_tokens': 728, 'pr...   \n","Luis923_Cremin516       {'token_usage': {'completion_tokens': 635, 'pr...   \n","Ronnie7_Greenfelder433  {'token_usage': {'completion_tokens': 583, 'pr...   \n","Tambra47_Lang846        {'token_usage': {'completion_tokens': 628, 'pr...   \n","Veronique514_Koepp521   {'token_usage': {'completion_tokens': 646, 'pr...   \n","Landon622_Beier427      {'token_usage': {'completion_tokens': 732, 'pr...   \n","Adrienne302_Zulauf375   {'token_usage': {'completion_tokens': 709, 'pr...   \n","\n","                                                          Patient_Summary  \\\n","Paulita78_Watsica258    ### Patient Summary\\n\\n**Patient Details:**\\n-...   \n","Grace552_Little434      ### Patient Summary\\n\\n**Patient Information:*...   \n","Dominique369_Daniel959  ### Patient Summary\\n\\n**Patient Details:**\\n-...   \n","Jeri234_Koss676         ### Patient Summary\\n\\n**Patient Details:**\\n-...   \n","Luis923_Cremin516       ### Patient Summary\\n\\n**Patient Details:**\\n-...   \n","Ronnie7_Greenfelder433  ### Patient Summary\\n\\n**Patient Details:**\\n-...   \n","Tambra47_Lang846        ### Patient Summary:\\n\\n**Patient Details:**\\n...   \n","Veronique514_Koepp521   ### Patient Summary\\n\\n**Patient Information:*...   \n","Landon622_Beier427      ### Patient Summary\\n\\n**Patient Details:**\\n-...   \n","Adrienne302_Zulauf375   ### Patient Summary\\n\\n**Patient Details:**\\n-...   \n","\n","                                                 Patient_Summary_Metadata  \\\n","Paulita78_Watsica258    {'token_usage': {'completion_tokens': 549, 'pr...   \n","Grace552_Little434      {'token_usage': {'completion_tokens': 646, 'pr...   \n","Dominique369_Daniel959  {'token_usage': {'completion_tokens': 518, 'pr...   \n","Jeri234_Koss676         {'token_usage': {'completion_tokens': 570, 'pr...   \n","Luis923_Cremin516       {'token_usage': {'completion_tokens': 495, 'pr...   \n","Ronnie7_Greenfelder433  {'token_usage': {'completion_tokens': 549, 'pr...   \n","Tambra47_Lang846        {'token_usage': {'completion_tokens': 507, 'pr...   \n","Veronique514_Koepp521   {'token_usage': {'completion_tokens': 557, 'pr...   \n","Landon622_Beier427      {'token_usage': {'completion_tokens': 535, 'pr...   \n","Adrienne302_Zulauf375   {'token_usage': {'completion_tokens': 647, 'pr...   \n","\n","                                                             Summary+Full  \\\n","Paulita78_Watsica258    Based on the provided clinical trial data and ...   \n","Grace552_Little434      Based on the patient's medical history and the...   \n","Dominique369_Daniel959  Based on the patient's medical history and the...   \n","Jeri234_Koss676         Based on the patient's medical history and the...   \n","Luis923_Cremin516       Based on the patient's medical history and the...   \n","Ronnie7_Greenfelder433  Based on the provided clinical trial data and ...   \n","Tambra47_Lang846        Based on the patient's medical history and the...   \n","Veronique514_Koepp521   Based on the patient's medical history and the...   \n","Landon622_Beier427      Based on the patient's profile and the clinica...   \n","Adrienne302_Zulauf375   Based on the patient's medical history and the...   \n","\n","                                                    Summary+Full_Metadata  \\\n","Paulita78_Watsica258    {'token_usage': {'completion_tokens': 485, 'pr...   \n","Grace552_Little434      {'token_usage': {'completion_tokens': 483, 'pr...   \n","Dominique369_Daniel959  {'token_usage': {'completion_tokens': 630, 'pr...   \n","Jeri234_Koss676         {'token_usage': {'completion_tokens': 472, 'pr...   \n","Luis923_Cremin516       {'token_usage': {'completion_tokens': 455, 'pr...   \n","Ronnie7_Greenfelder433  {'token_usage': {'completion_tokens': 498, 'pr...   \n","Tambra47_Lang846        {'token_usage': {'completion_tokens': 561, 'pr...   \n","Veronique514_Koepp521   {'token_usage': {'completion_tokens': 459, 'pr...   \n","Landon622_Beier427      {'token_usage': {'completion_tokens': 514, 'pr...   \n","Adrienne302_Zulauf375   {'token_usage': {'completion_tokens': 554, 'pr...   \n","\n","                                                              Summary+RAG  \\\n","Paulita78_Watsica258    Based on the provided patient summary and the ...   \n","Grace552_Little434      Based on the provided patient summary and the ...   \n","Dominique369_Daniel959  Based on the provided patient summary and the ...   \n","Jeri234_Koss676         Based on the provided patient summary and the ...   \n","Luis923_Cremin516       Based on the provided patient summary and the ...   \n","Ronnie7_Greenfelder433  Based on the provided patient summary and the ...   \n","Tambra47_Lang846        Based on the provided patient summary and the ...   \n","Veronique514_Koepp521   Based on the provided patient summary and the ...   \n","Landon622_Beier427      Based on the provided patient summary and the ...   \n","Adrienne302_Zulauf375   Based on the provided patient summary and the ...   \n","\n","                                                     Summary+RAG_Metadata  \\\n","Paulita78_Watsica258    ChatCompletion(id='chatcmpl-9ra1QpLIgojeY1b2x0...   \n","Grace552_Little434      ChatCompletion(id='chatcmpl-9ra1mdoBEbSAGZdR1A...   \n","Dominique369_Daniel959  ChatCompletion(id='chatcmpl-9ra2K1kOpFxcptRDHh...   \n","Jeri234_Koss676         ChatCompletion(id='chatcmpl-9ra2zVlZ8SA3dk7IuT...   \n","Luis923_Cremin516       ChatCompletion(id='chatcmpl-9ra3OvWghmawTBdDdB...   \n","Ronnie7_Greenfelder433  ChatCompletion(id='chatcmpl-9ra4bgbodPx4WtONI3...   \n","Tambra47_Lang846        ChatCompletion(id='chatcmpl-9ra4vJnwufai3TkUYZ...   \n","Veronique514_Koepp521   ChatCompletion(id='chatcmpl-9ra5Lr5pVN6kMqeJJI...   \n","Landon622_Beier427      ChatCompletion(id='chatcmpl-9ra5oOtF2GwEp3cZI6...   \n","Adrienne302_Zulauf375   ChatCompletion(id='chatcmpl-9ra69b7GIZkrdxDMAr...   \n","\n","                                                     Summary+AlternateRAG  \\\n","Paulita78_Watsica258    Based on the provided patient summary and the ...   \n","Grace552_Little434      Based on the provided patient summary and the ...   \n","Dominique369_Daniel959  Based on the provided patient summary and the ...   \n","Jeri234_Koss676         Based on the provided patient summary and the ...   \n","Luis923_Cremin516       Based on the provided patient summary and the ...   \n","Ronnie7_Greenfelder433  Based on the provided patient summary and the ...   \n","Tambra47_Lang846        Based on the provided patient summary and the ...   \n","Veronique514_Koepp521   Based on the provided patient summary and the ...   \n","Landon622_Beier427      Based on the provided patient summary and the ...   \n","Adrienne302_Zulauf375   Based on the provided patient summary and the ...   \n","\n","                                            Summary+AlternateRAG_Metadata  \n","Paulita78_Watsica258    ChatCompletion(id='chatcmpl-9raOIcPoUUoSnPMy2O...  \n","Grace552_Little434      ChatCompletion(id='chatcmpl-9raOoaVlUd8ckGQpYt...  \n","Dominique369_Daniel959  ChatCompletion(id='chatcmpl-9raPVbh3aImZH6x4JQ...  \n","Jeri234_Koss676         ChatCompletion(id='chatcmpl-9raQ0ekcc9zGHD1IAN...  \n","Luis923_Cremin516       ChatCompletion(id='chatcmpl-9raQISTnB7yo2T3Wqd...  \n","Ronnie7_Greenfelder433  ChatCompletion(id='chatcmpl-9raQl3Aqc7yTt7Skp2...  \n","Tambra47_Lang846        ChatCompletion(id='chatcmpl-9raR8Fi4HK9wAqNEVh...  \n","Veronique514_Koepp521   ChatCompletion(id='chatcmpl-9raRdubZZDZF0scf51...  \n","Landon622_Beier427      ChatCompletion(id='chatcmpl-9raS6ku9m35wv4XIMP...  \n","Adrienne302_Zulauf375   ChatCompletion(id='chatcmpl-9raSkjO5SeTBPXytXH...  \n"]}],"source":["print(llm_responses_df)"]},{"cell_type":"markdown","source":["References:\n","\n","OpenAI et. al. (2023). GPT-4 Technical Report. https://arxiv.org/abs/2303.08774\n","\n","OpenAI. (2024). Hello GPT-4o. https://openai.com/index/hello-gpt-4o/\n","Luo et. al. (2022). BioGPT: Generative Pre-trained Transformer for Biomedical Text Generation and Mining. https://arxiv.org/abs/2210.10341\n","Clinicaltrials.gov.\n","\n","NIH: User Guide for Clinicaltrials.gov Website: https://clinicaltrials.gov/submit-studies/prs-help/user-guide#intro\n","\n","Jin, Q., Wang, Z., Floudas, C. S., Chen, F., Gong, C., Bracken-Clarke, D., ... & Lu, Z. (2023). Matching patients to clinical trials with large language models. ArXiv.arXiv:2307.15051v4. Accessed on August 2, 2024. Available from: https://www.ncbi.nlm.nih.gov/pmc/articles/PMC10418514/\n","Koopman, B., & Zuccon, G. (2016, July). A test collection for matching patients to clinical trials. In Proceedings of the 39th International ACM SIGIR conference on Research and Development in Information Retrieval (pp. 669-672).\n","\n","Msv, J. (2023, July 21). Tutorial: Build a Q&A bot for Academy Awards based on ChatGPT. The New Stack. https://thenewstack.io/tutorial-build-a-qa-bot-for-academy-awards-based-on-chatgpt/\n","Stack OverFlow, Accessed on August 4th 2024. https://stackoverflow.com/questions/78415818/how-to-get-full-results-with-clinicaltrials-gov-api-in-python\n","\n","Roberts, K., Demner-Fushman, D., Voorhees, E.M., Bedrick, S. & Hersh, W.R. Overview of the TREC 2021 Clinical Trials Track. in Proceedings of the Thirtieth Text REtrieval Conference (TREC 2021) (2021).\n","\n","Walonoski, J., Kramer, M., Nichols, J., Quina, A., Moesel, C., Hall, D., ... & McLachlan, S. (2018). Synthea: An approach, method, and software mechanism for generating synthetic patients and the synthetic electronic health care record. Journal of the American Medical Informatics Association, 25(3), 230-238.\n","\n","Ceylan, B., & Özerdoğan, N. (2015). Factors affecting age of onset of menopause and determination of quality of life in menopause. Turkish journal of obstetrics and gynecology, 12(1), 43–49. https://doi.org/10.4274/tjod.79836\n","Swanner, K. D., & Richmond, L. B. (2023). A 65-Year-Old Woman With No Menopause History: A Case Report. Cureus, 15(9), e44792. https://doi.org/10.7759/cureus.44792\n"],"metadata":{"id":"m0dus1tsnm1a"}}],"metadata":{"colab":{"provenance":[]},"kernelspec":{"display_name":"Python 3","name":"python3"},"language_info":{"name":"python"}},"nbformat":4,"nbformat_minor":0}