Web Crawling Questions and Answer App using GPT Embedding.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/anshupandey/Generative-AI-and-Prompt-Engineering/blob/main/Web%20Crawling%20Questions%20and%20Answer%20App%20using%20GPT%20Embedding.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "S9Jy4LDTSPbP",
        "outputId": "49e6e8a3-9534-4078-fd41-898aa27997b0"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "https://openai.com/\n",
            "https://openai.com/security\n",
            "https://openai.com/policies/api-data-usage-policies\n",
            "https://openai.com/enterprise-privacy#content\n",
            "https://openai.com/enterprise-privacy#our-commitments\n",
            "https://openai.com/careers/software-engineer-privacy\n",
            "https://openai.com/careers/software-engineer-privacy#content\n",
            "https://openai.com/security#content\n",
            "https://openai.com/blog/bug-bounty-program\n",
            "https://openai.com/blog?authors=openai\n",
            "https://openai.com/blog/announcing-openai-devday\n",
            "https://openai.com/blog/announcing-openai-devday#OpenAI\n",
            "https://openai.com/blog/introducing-chatgpt-and-whisper-apis\n",
            "https://openai.com/blog/introducing-chatgpt-and-whisper-apis#RachelLim\n",
            "https://openai.com/blog/introducing-chatgpt-and-whisper-apis#LukeMiller\n",
            "https://openai.com/blog/introducing-chatgpt-and-whisper-apis#AttyEleti\n",
            "https://openai.com/blog/introducing-chatgpt-and-whisper-apis#JoanneJang\n",
            "https://openai.com/blog?authors=joanne-jang\n",
            "https://openai.com/blog/introducing-chatgpt-and-whisper-apis#MichellePokrass\n",
            "https://openai.com/blog?topics=product\n",
            "https://openai.com/blog?authors=atty-eleti\n",
            "https://openai.com/blog/introducing-chatgpt-and-whisper-apis#content\n",
            "https://openai.com/blog/introducing-chatgpt-and-whisper-apis#GregBrockman\n",
            "https://openai.com/blog?authors=elie-georges\n",
            "https://openai.com/blog?authors=luke-miller\n",
            "https://openai.com/blog?authors=greg-brockman\n",
            "https://openai.com/blog?authors=logan-kilpatrick\n",
            "https://openai.com/blog?authors=michelle-pokrass\n",
            "https://openai.com/blog/whisper\n",
            "https://openai.com/research?models=whisper\n",
            "https://openai.com/research/gpts-are-gpts\n",
            "https://openai.com/research/gpts-are-gpts#content\n",
            "https://openai.com/research?models=gpt-4\n",
            "https://openai.com/research/scaling-laws-for-reward-model-overoptimization\n",
            "https://openai.com/research?authors=leo-gao\n",
            "https://openai.com/research?authors=john-schulman-2\n",
            "https://openai.com/research?authors=jacob-hilton\n",
            "https://openai.com/research/scaling-laws-for-reward-model-overoptimization#content\n",
            "https://openai.com/research?topics=image-generation\n",
            "https://openai.com/research/dall-e-2-pre-training-mitigations\n",
            "https://openai.com/research/dall-e-2-pre-training-mitigations#content\n",
            "https://openai.com/research?models=dall-e-2\n",
            "https://openai.com/research?contentTypes=conclusion\n",
            "https://openai.com/dall-e-2\n",
            "https://openai.com/research/clip\n",
            "https://openai.com/research?authors=ilya-sutskever\n",
            "https://openai.com/research?authors=sandhini-agarwal\n",
            "https://openai.com/research?topics=supervised-learning\n",
            "https://openai.com/research?models=clip\n",
            "https://openai.com/research?topics=contrastive-learning\n",
            "https://openai.com/research?topics=transfer-learning\n",
            "https://openai.com/research?authors=gretchen-krueger\n",
            "https://openai.com/research?authors=alec-radford\n",
            "https://openai.com/research?authors=jong-wook-kim\n",
            "https://openai.com/research/clip#content\n",
            "https://openai.com/research/dall-e\n",
            "https://openai.com/research?models=gpt-2\n",
            "https://openai.com/research?authors=vedant-misra\n",
            "https://openai.com/research/dall-e#content\n",
            "https://openai.com/research?authors=aditya-ramesh\n",
            "https://openai.com/research?authors=rewon-child\n",
            "https://openai.com/research?authors=pamela-mishkin\n",
            "https://openai.com/research?authors=mark-chen\n",
            "https://openai.com/blog/image-gpt\n",
            "https://openai.com/research/image-gpt#experimental-results\n",
            "https://openai.com/jobs\n",
            "https://openai.com/careers/full-stack-engineer-3rd-party-data-2\n",
            "https://openai.com/careers/full-stack-engineer-3rd-party-data-2#content\n",
            "https://openai.com/blog/webgpt\n",
            "https://openai.com/blog/debate\n",
            "https://openai.com/research/debate#content\n",
            "https://openai.com/research/concrete-ai-safety-problems\n",
            "https://openai.com/research?topics=robustness\n",
            "https://openai.com/research?authors=paul-christiano\n",
            "https://openai.com/research/concrete-ai-safety-problems#content\n",
            "https://openai.com/research?authors=greg-brockman\n",
            "https://openai.com/research?authors=geoffrey-irving\n",
            "https://openai.com/research?authors=dario-amodei\n",
            "https://openai.com/research/learning-from-human-preferences\n",
            "https://openai.com/research?authors=alex-ray\n",
            "https://openai.com/blog/deep-reinforcement-learning-from-human-preferences/#bflip\n",
            "https://openai.com/research/learning-from-human-preferences#content\n",
            "https://openai.com/research/dota-2\n",
            "https://openai.com/the-international\n",
            "HTTP Error 404: Not Found\n",
            "https://openai.com/research?topics=dota-2\n",
            "https://openai.com/research?models=openai-five\n",
            "https://openai.com/research?authors=openai\n",
            "https://openai.com/research?topics=self-play\n",
            "https://openai.com/research/dota-2#content\n",
            "https://openai.com/research/attacking-machine-learning-with-adversarial-examples\n",
            "https://openai.com/research?authors=sandy-huang\n",
            "https://openai.com/blog/openai-technical-goals\n",
            "https://openai.com/blog/openai-technical-goals#IlyaSutskever\n",
            "https://openai.com/blog/openai-technical-goals#ElonMusk\n",
            "https://openai.com/blog?authors=ilya-sutskever\n",
            "https://openai.com/blog?topics=research\n",
            "https://openai.com/blog/openai-technical-goals#GregBrockman\n",
            "https://openai.com/blog?authors=sam-altman\n",
            "https://openai.com/blog/openai-technical-goals#SamAltman\n",
            "https://openai.com/blog?authors=elon-musk\n",
            "https://openai.com/blog/generative-models\n",
            "https://openai.com/research?authors=vicki-cheung\n",
            "https://openai.com/research/generative-models#content\n",
            "https://openai.com/research?authors=rein-houthooft\n",
            "https://openai.com/blog/generative-models/#going-forward\n",
            "https://openai.com/research?authors=peter-chen\n",
            "https://openai.com/blog/openai-gym-beta\n",
            "https://openai.com/research?topics=robotics\n",
            "https://openai.com/blog/welcome-pieter-and-shivon\n",
            "https://openai.com/blog/welcome-pieter-and-shivon#IlyaSutskever\n",
            "https://openai.com/blog/welcome-pieter-and-shivon#content\n",
            "https://openai.com/research?topics=environments\n",
            "https://openai.com/research/openai-gym-beta#content\n",
            "https://openai.com/blog/team-plus-plus\n",
            "https://openai.com/blog/team-plus-plus#content\n",
            "https://openai.com/blog/team-plus-plus#GregBrockman\n",
            "https://openai.com/research?authors=andrej-karpathy\n",
            "https://openai.com/research?authors=jonathan-ho\n",
            "https://openai.com/research?authors=tim-salimans\n",
            "https://openai.com/research?authors=durk-kingma\n",
            "https://openai.com/research?authors=wojciech-zaremba\n",
            "https://openai.com/blog/openai-technical-goals#content\n",
            "https://openai.com/research?authors=jack-clark\n",
            "https://openai.com/research?authors=nicolas-papernot\n",
            "https://openai.com/research?authors=pieter-abbeel\n",
            "https://openai.com/research/attacking-machine-learning-with-adversarial-examples#content\n",
            "https://openai.com/research?authors=ian-goodfellow\n",
            "https://openai.com/research?authors=yan-duan\n",
            "https://openai.com/research?topics=adversarial-examples\n",
            "https://openai.com/research?authors=reiichiro-nakano\n",
            "https://openai.com/blog/fine-tuning-gpt-2\n",
            "https://openai.com/research?authors=jeffrey-wu\n",
            "https://openai.com/research/learning-complex-goals-with-iterated-amplification\n",
            "https://openai.com/research/faulty-reward-functions\n",
            "https://openai.com/research/faulty-reward-functions#content\n",
            "https://openai.com/blog/concrete-ai-safety-problems\n",
            "https://openai.com/research/learning-complex-goals-with-iterated-amplification#content\n",
            "https://openai.com/research/debate\n",
            "https://openai.com/research?authors=daniel-ziegler\n",
            "https://openai.com/research/gpt-2-6-month-follow-up\n",
            "https://openai.com/research/gpt-2-6-month-follow-up#content\n",
            "https://openai.com/blog/better-language-models\n",
            "https://openai.com/research?authors=david-luan\n",
            "https://openai.com/research?authors=daniella-amodei\n",
            "https://openai.com/research?authors=david-lansky\n",
            "https://openai.com/research/better-language-models#content\n",
            "https://openai.com/research?authors=miles-brundage\n",
            "https://openai.com/research/preparing-for-malicious-uses-of-ai\n",
            "https://openai.com/research/preparing-for-malicious-uses-of-ai#content\n",
            "https://openai.com/research?authors=michael-page\n",
            "https://openai.com/research/language-unsupervised\n",
            "https://openai.com/research/unsupervised-sentiment-neuron\n",
            "https://openai.com/blog/unsupervised-sentiment-neuron/#sentimentneuron\n",
            "https://openai.com/blog/unsupervised-sentiment-neuron/#methodology\n",
            "https://openai.com/research?authors=rafal-jozefowicz\n",
            "https://openai.com/research/unsupervised-sentiment-neuron#content\n",
            "https://openai.com/research?models=gpt\n",
            "https://openai.com/research/language-unsupervised#content\n",
            "https://openai.com/research?authors=danny-hernandez\n",
            "https://openai.com/research?authors=amanda-askell\n",
            "https://openai.com/blog/better-language-models/#update\n",
            "https://openai.com/research?authors=nisan-stiennon\n",
            "https://openai.com/research?authors=tom-brown\n",
            "https://openai.com/blog/amplifying-ai-training\n",
            "https://openai.com/research/fine-tuning-gpt-2#content\n",
            "https://openai.com/research?authors=suchir-balaji\n",
            "https://openai.com/blog/webgpt/#samples\n",
            "https://openai.com/blog/learning-to-summarize-with-human-feedback\n",
            "https://openai.com/research?authors=ryan-lowe\n",
            "https://openai.com/blog/openai-baselines-ppo\n",
            "https://openai.com/research?topics=policy-optimization\n",
            "https://openai.com/research?authors=filip-wolski\n",
            "https://openai.com/research?authors=prafulla-dhariwal\n",
            "https://openai.com/research?authors=oleg-klimov\n",
            "https://openai.com/research/roboschool\n",
            "https://openai.com/research/roboschool#content\n",
            "https://openai.com/research/openai-baselines-ppo#content\n",
            "https://openai.com/blog/learning-to-summarize-with-human-feedback/#rf1\n",
            "https://openai.com/blog/learning-to-summarize-with-human-feedback/#rf7\n",
            "https://openai.com/research?authors=chelsea-voss\n",
            "https://openai.com/research?authors=long-ouyang\n",
            "https://openai.com/blog/learning-to-summarize-with-human-feedback/#rf20\n",
            "https://openai.com/research/learning-to-summarize-with-human-feedback#content\n",
            "https://openai.com/research/webgpt#content\n",
            "https://openai.com/blog/summarizing-books\n",
            "https://openai.com/research/summarizing-books#content\n",
            "https://openai.com/research?authors=jan-leike\n",
            "https://openai.com/blog/deep-reinforcement-learning-from-human-preferences\n",
            "https://openai.com/research?models=gpt-3\n",
            "https://openai.com/blog/webgpt/#rf4\n",
            "https://openai.com/blog/instruction-following\n",
            "https://openai.com/research/instruction-following#content\n",
            "https://openai.com/blog/improving-language-model-behavior\n",
            "https://openai.com/research?authors=christy-dennison\n",
            "https://openai.com/research/improving-language-model-behavior#content\n",
            "https://openai.com/research?authors=irene-solaiman\n",
            "https://openai.com/blog/gpt-3-apps\n",
            "https://openai.com/blog/gpt-3-apps#content\n",
            "https://openai.com/blog/gpt-3-apps#OpenAI\n",
            "https://openai.com/blog?authors=ashley-pilipiszyn\n",
            "https://openai.com/blog/requests-for-research-2\n",
            "https://openai.com/research/requests-for-research-2#content\n",
            "https://openai.com/research/competitive-self-play\n",
            "https://openai.com/research/competitive-self-play#content\n",
            "https://openai.com/research/more-on-dota-2\n",
            "https://openai.com/research/more-on-dota-2#content\n",
            "https://openai.com/research?authors=szymon-sidor\n",
            "https://openai.com/research?authors=igor-mordatch\n",
            "https://openai.com/research?authors=jakub-pachocki\n",
            "https://openai.com/research?authors=trapit-bansal\n",
            "https://openai.com/blog/openai-api\n",
            "https://openai.com/blog/openai-api#OpenAI\n",
            "https://openai.com/blog/openai-api#content\n",
            "https://openai.com/blog/openai-lp\n",
            "https://openai.com/blog/openai-lp#GregBrockman\n",
            "https://openai.com/blog/openai-fellows-interns-2019\n",
            "https://openai.com/blog?authors=larissa-schiavo\n",
            "https://openai.com/blog?topics=culture-and-careers\n",
            "https://openai.com/blog/openai-fellows-interns-2019#LarissaSchiavo\n",
            "https://openai.com/blog/openai-fellows-interns-2019#content\n",
            "https://openai.com/blog/openai-fellows-interns-2019#AshleyPilipiszyn\n",
            "https://openai.com/blog/openai-five\n",
            "https://openai.com/research?authors=shariq-hashme\n",
            "https://openai.com/research/openai-five#observations\n",
            "https://openai.com/research?authors=jonas-schneider\n",
            "https://openai.com/research/openai-five#thegames\n",
            "https://openai.com/research?authors=diane-yoon\n",
            "https://openai.com/research?authors=susan-zhang\n",
            "https://openai.com/research/more-on-dota-2#the-task\n",
            "https://openai.com/research?authors=quirin-fischer\n",
            "https://openai.com/research?authors=brooke-chan\n",
            "https://openai.com/research?authors=przemyslaw-debiak\n",
            "https://openai.com/research?authors=larissa-schiavo\n",
            "https://openai.com/research?authors=henrique-ponde\n",
            "https://openai.com/research?authors=jie-tang\n",
            "https://openai.com/research?authors=jonathan-raiman\n",
            "https://openai.com/research/openai-five#exploration\n",
            "https://openai.com/research/openai-five#the-problem\n",
            "https://openai.com/research/generalizing-from-simulation\n",
            "https://openai.com/blog/generalizing-from-simulation/#pusher_fail\n",
            "https://openai.com/research?authors=marcin-andrychowicz\n",
            "https://openai.com/research?authors=peter-welinder\n",
            "https://openai.com/research?authors=josh-tobin\n",
            "https://openai.com/research?authors=lerrel-pinto\n",
            "https://openai.com/research?authors=bob-mcgrew\n",
            "https://openai.com/research?topics=domain-randomization\n",
            "https://openai.com/research?topics=sim-to-real\n",
            "https://openai.com/research?authors=xue-bin-peng\n",
            "https://openai.com/research/generalizing-from-simulation#content\n",
            "https://openai.com/research?authors=christopher-hesse\n",
            "https://openai.com/research/more-on-dota-2#infrastructure\n",
            "https://openai.com/research/spam-detection-in-the-physical-world\n",
            "https://openai.com/research/spam-detection-in-the-physical-world#content\n",
            "https://openai.com/research?authors=rachel-fong\n",
            "https://openai.com/research/learning-a-hierarchy\n",
            "https://openai.com/research/learning-a-hierarchy#content\n",
            "https://openai.com/research?topics=meta-learning\n",
            "https://openai.com/research?authors=kevin-frans\n",
            "https://openai.com/research?authors=david-farhi\n",
            "https://openai.com/research?authors=eric-sigler\n",
            "https://openai.com/research/openai-five#restricted\n",
            "https://openai.com/research/openai-baselines-ppo\n",
            "https://openai.com/research?authors=michael-petrov\n",
            "https://openai.com/research?authors=christopher-berner\n",
            "https://openai.com/research/openai-five#content\n",
            "https://openai.com/blog/openai-lp#IlyaSutskever\n",
            "https://openai.com/blog/openai-lp#OpenAI\n",
            "https://openai.com/blog/openai-supporters\n",
            "https://openai.com/blog/openai-supporters#content\n",
            "https://openai.com/research/robots-that-learn\n",
            "https://openai.com/research?authors=vikash-kumar\n",
            "https://openai.com/research?authors=bradly-stadie\n",
            "https://openai.com/research/robots-that-learn#content\n",
            "https://openai.com/research?authors=ankur-handa\n",
            "https://openai.com/research?authors=erika-reinhardt\n",
            "https://openai.com/research?authors=matthias-plappert\n",
            "https://openai.com/blog/openai-lp#content\n",
            "https://openai.com/blog/openai-scholars\n",
            "https://openai.com/blog/openai-scholars#LarissaSchiavo\n",
            "https://openai.com/blog/openai-scholars#content\n",
            "https://openai.com/blog/openai-scholars#GregBrockman\n",
            "https://openai.com/blog/ai-and-compute\n",
            "https://openai.com/blog/ai-and-compute/#lookingforward\n",
            "https://openai.com/research/ai-and-compute#appendixrecentnovelresultsthatusedmodestamountsofcompute\n",
            "https://openai.com/research?authors=girish-sastry\n",
            "https://openai.com/blog/ai-and-compute/#modern\n",
            "https://openai.com/research/ai-and-compute#content\n",
            "https://openai.com/blog/learning-dexterity\n",
            "https://openai.com/research/learning-dexterity#content\n",
            "https://openai.com/research?authors=ben-barry\n",
            "https://openai.com/research?authors=lilian-weng\n",
            "https://openai.com/research?authors=maciek-chociej\n",
            "https://openai.com/research?models=dactyl\n",
            "https://openai.com/research/ingredients-for-robotics-research\n",
            "https://openai.com/research/ingredients-for-robotics-research#content\n",
            "https://openai.com/research/requests-for-research-2\n",
            "https://openai.com/research/openai-baselines-dqn\n",
            "https://openai.com/research/openai-baselines-dqn#content\n",
            "https://openai.com/blog/openai-baselines-dqn/#dqn\n",
            "https://openai.com/research/openai-five\n",
            "https://openai.com/research?authors=bowen-baker\n",
            "https://openai.com/research/openai-five#rapid\n",
            "https://openai.com/research?authors=glenn-powell\n",
            "https://openai.com/blog/learning-dexterity/#results\n",
            "https://openai.com/research/ai-and-compute\n",
            "https://openai.com/research?authors=arthur-petron\n",
            "https://openai.com/blog/openai-api#GregBrockman\n",
            "https://openai.com/blog/openai-api#MiraMurati\n",
            "https://openai.com/blog?authors=mira-murati\n",
            "https://openai.com/blog/openai-api#PeterWelinder\n",
            "https://openai.com/blog?authors=peter-welinder\n",
            "https://openai.com/blog/gpt-3-apps#AshleyPilipiszyn\n",
            "https://openai.com/jobs/#open\n",
            "https://openai.com/blog/the-power-of-continuous-learning\n",
            "https://openai.com/blog/new-and-improved-content-moderation-tooling\n",
            "https://openai.com/blog?authors=steven-adler\n",
            "https://openai.com/blog?authors=todor-markov\n",
            "https://openai.com/blog/best-practices-for-deploying-language-models\n",
            "https://openai.com/blog?topics=safety-alignment\n",
            "https://openai.com/blog/best-practices-for-deploying-language-models#content\n",
            "https://openai.com/blog/best-practices-for-deploying-language-models#OpenAI\n",
            "https://openai.com/blog/new-and-improved-content-moderation-tooling#StevenAdler\n",
            "https://openai.com/blog?authors=chong-zhang\n",
            "https://openai.com/blog?authors=sandhini-agarwal\n",
            "https://openai.com/blog?authors=teddy-lee\n",
            "https://openai.com/blog/new-and-improved-content-moderation-tooling#TodorMarkov\n",
            "https://openai.com/blog/new-and-improved-content-moderation-tooling#TeddyLee\n",
            "https://openai.com/blog/customized-gpt-3\n",
            "https://openai.com/blog/customizing-gpt-3#LukeMiller\n",
            "https://openai.com/blog/customizing-gpt-3#content\n",
            "https://openai.com/blog/customizing-gpt-3#example\n",
            "https://openai.com/blog/customizing-gpt-3#MichaelWu\n",
            "https://openai.com/blog/grade-school-math\n",
            "https://openai.com/research/solving-math-word-problems#content\n",
            "https://openai.com/research?authors=karl-cobbe\n",
            "https://openai.com/blog/grade-school-math/#samples\n",
            "https://openai.com/research?authors=vineet-kosaraju\n",
            "https://openai.com/blog/customizing-gpt-3#RachelLim\n",
            "https://openai.com/blog?authors=michael-wu\n",
            "https://openai.com/api\n",
            "https://openai.com/safety-standards\n",
            "https://openai.com/blog/new-ai-classifier-for-indicating-ai-written-text\n",
            "https://openai.com/blog?authors=jan-leike\n",
            "https://openai.com/blog/new-ai-classifier-for-indicating-ai-written-text#content\n",
            "https://openai.com/blog?authors=scott-aaronson\n",
            "https://openai.com/blog/forecasting-misuse\n",
            "https://openai.com/research/forecasting-misuse#content\n",
            "https://openai.com/blog/new-ai-classifier-for-indicating-ai-written-text#JanHendrik Kirchner\n",
            "https://openai.com/blog/new-ai-classifier-for-indicating-ai-written-text#LamaAhmad\n",
            "https://openai.com/blog?authors=jan-hendrik-kirchner\n",
            "https://openai.com/blog?authors=lama-ahmad\n",
            "https://openai.com/blog/new-ai-classifier-for-indicating-ai-written-text#ScottAaronson\n",
            "https://openai.com/blog/new-ai-classifier-for-indicating-ai-written-text#JanLeike\n",
            "https://openai.com/policies/usage-policies\n",
            "https://openai.com/safety-standards#content\n",
            "https://openai.com/customer-stories/government-of-iceland\n",
            "https://openai.com/customer-stories/government-of-iceland#content\n",
            "https://openai.com/customer-stories?topics=language\n",
            "https://openai.com/customer-stories/inworld-ai\n",
            "https://openai.com/customer-stories/inworld-ai#content\n",
            "https://openai.com/customer-stories/khan-academy\n",
            "https://openai.com/customer-stories/khan-academy#content\n",
            "https://openai.com/customer-stories#content\n",
            "https://openai.com/customer-stories/morgan-stanley\n",
            "https://openai.com/customer-stories/morgan-stanley#content\n",
            "https://openai.com/customer-stories/waymark\n",
            "https://openai.com/customer-stories/waymark#content\n",
            "https://openai.com/customer-stories/yabble\n",
            "https://openai.com/customer-stories/yabble#content\n",
            "https://openai.com/customer-stories/be-my-eyes\n",
            "https://openai.com/customer-stories/be-my-eyes#content\n",
            "https://openai.com/customer-stories/duolingo\n",
            "https://openai.com/customer-stories/duolingo#content\n",
            "https://openai.com/customer-stories/viable\n",
            "https://openai.com/customer-stories/viable#content\n",
            "https://openai.com/customer-stories/stripe\n",
            "https://openai.com/customer-stories/stripe#content\n",
            "https://openai.com/product#content\n",
            "https://openai.com/product#made-for-developers\n",
            "https://openai.com/blog/new-and-improved-content-moderation-tooling#AngelaJiang\n",
            "https://openai.com/blog/critiques\n",
            "https://openai.com/research/critiques#content\n",
            "https://openai.com/research?authors=catherine-yeh\n",
            "https://openai.com/research?authors=william-saunders\n",
            "https://openai.com/blog/new-and-improved-content-moderation-tooling#LilianWeng\n",
            "https://openai.com/blog?authors=angela-jiang\n",
            "https://openai.com/blog/new-and-improved-content-moderation-tooling#ChongZhang\n",
            "https://openai.com/blog?authors=tyna-eloundou\n",
            "https://openai.com/blog/new-and-improved-content-moderation-tooling#TynaEloundou\n",
            "https://openai.com/blog/new-and-improved-content-moderation-tooling#SandhiniAgarwal\n",
            "https://openai.com/blog?authors=lilian-weng\n",
            "https://openai.com/blog/new-and-improved-content-moderation-tooling#content\n",
            "https://openai.com/blog/the-power-of-continuous-learning#OpenAI\n",
            "https://openai.com/blog/the-power-of-continuous-learning#content\n",
            "https://openai.com/blog/solving-rubiks-cube\n",
            "https://openai.com/research?authors=nikolas-tezak\n",
            "https://openai.com/research?authors=mateusz-litwin\n",
            "https://openai.com/blog/solving-rubiks-cube/#perturbations\n",
            "https://openai.com/research?authors=alex-paino\n",
            "https://openai.com/blog/solving-rubiks-cube/#challenges\n",
            "https://openai.com/research/solving-rubiks-cube#content\n",
            "https://openai.com/research?authors=raphael-ribas\n",
            "https://openai.com/research?authors=jerry-tworek\n",
            "https://openai.com/jobs/#robotics\n",
            "https://openai.com/research?authors=ilge-akkaya\n",
            "https://openai.com/research?authors=lei-zhang\n",
            "https://openai.com/blog/how-to-train-your-openai-five\n",
            "https://openai.com/research/openai-five#our-approach\n",
            "https://openai.com/research/openai-five-benchmark-results#training\n",
            "https://openai.com/research/openai-five-benchmark-results#content\n",
            "https://openai.com/blog/openai-five-finals\n",
            "https://openai.com/blog/the-international-2018-results\n",
            "https://openai.com/blog/openai-five-benchmark#restrictions\n",
            "https://openai.com/blog/openai-five-benchmark#content\n",
            "https://openai.com/blog/openai-five-benchmark/#restrictions\n",
            "https://openai.com/blog/openai-five-benchmark-results\n",
            "https://openai.com/blog/openai-five-benchmark#OpenAI\n",
            "https://openai.com/research/the-international-2018-results#content\n",
            "https://openai.com/research/the-international-2018-results#rules-change\n",
            "https://openai.com/five/#overview\n",
            "https://openai.com/blog?topics=events\n",
            "https://openai.com/blog/openai-five-finals#OpenAI\n",
            "https://openai.com/blog/openai-five-finals#content\n",
            "https://openai.com/research/openai-five-defeats-dota-2-world-champions#replays\n",
            "https://openai.com/research/more-on-dota-2#bot-exploits\n",
            "https://openai.com/research/openai-five-defeats-dota-2-world-champions#content\n",
            "https://openai.com/research/retro-contest-results\n",
            "https://openai.com/blog/retro-contest\n",
            "https://openai.com/research/retro-contest#content\n",
            "https://openai.com/research/retro-contest-results#content\n",
            "https://openai.com/research?authors=vicki-pfau\n",
            "https://openai.com/research/learning-dexterity\n",
            "https://openai.com/blog/openai-five-benchmark\n",
            "https://openai.com/research?authors=qiming-yuan\n",
            "https://openai.com/careers/sales-engineer-dublin\n",
            "https://openai.com/careers/sales-engineer-dublin#content\n",
            "https://openai.com/blog/openai-residency\n",
            "https://openai.com/blog/openai-residency#content\n",
            "https://openai.com/blog/musenet\n",
            "https://openai.com/blog/sparse-transformer\n",
            "https://openai.com/research/sparse-transformer#content\n",
            "https://openai.com/research?topics=sparsity\n",
            "https://openai.com/blog/block-sparse-gpu-kernels\n",
            "https://openai.com/research/block-sparse-gpu-kernels#content\n",
            "https://openai.com/research/block-sparse-gpu-kernels#small-world-lstms\n",
            "https://openai.com/research/block-sparse-gpu-kernels#small-world\n",
            "https://openai.com/research?topics=audio-generation\n",
            "https://openai.com/research/musenet#content\n",
            "https://openai.com/research?models=musenet\n",
            "https://openai.com/blog/openai-residency#OpenAI\n",
            "https://openai.com/blog/discovering-the-minutiae-of-backend-systems\n",
            "https://openai.com/blog/discovering-the-minutiae-of-backend-systems#OpenAI\n",
            "https://openai.com/blog/discovering-the-minutiae-of-backend-systems#content\n",
            "https://openai.com/careers/research-engineer-fine-tuning-api\n",
            "https://openai.com/careers/research-engineer-fine-tuning-api#content\n",
            "https://openai.com/careers/it-support\n",
            "https://openai.com/careers/it-support#content\n",
            "https://openai.com/interview-guide\n",
            "https://openai.com/interview-guide#content\n",
            "https://openai.com/commitment-to-dei\n",
            "https://openai.com/commitment-to-dei#content\n",
            "https://openai.com/careers/media-relations-europe-lead-2\n",
            "https://openai.com/careers/media-relations-europe-lead-2#content\n",
            "https://openai.com/careers#content\n",
            "https://openai.com/research/image-gpt#from-language-gpt-to-image-gpt\n",
            "https://openai.com/research?topics=unsupervised-learning\n",
            "https://openai.com/blog/image-gpt/#completions\n",
            "https://openai.com/blog/image-gpt/#samples\n",
            "https://openai.com/research/image-gpt#introduction\n",
            "https://openai.com/research/image-gpt#content\n",
            "https://openai.com/blog/image-gpt/#rf24\n",
            "https://openai.com/research/image-gpt#approach\n",
            "https://openai.com/research/image-gpt#towards-general-unsupervised-learning\n",
            "https://openai.com/research/image-gpt#limitations\n",
            "https://openai.com/research/image-gpt#conclusion\n",
            "https://openai.com/research?models=dall-e\n",
            "https://openai.com/research?authors=gabriel-goh\n",
            "https://openai.com/research?authors=mikhail-pavlov\n",
            "https://openai.com/research?authors=scott-gray\n",
            "https://openai.com/blog/dall-e-introducing-outpainting\n",
            "https://openai.com/blog/dall-e-introducing-outpainting#content\n",
            "https://openai.com/blog/dall-e-introducing-outpainting#OpenAI\n",
            "https://openai.com/blog/dall-e-2-extending-creativity\n",
            "https://openai.com/blog?topics=community\n",
            "https://openai.com/blog/dall-e-2-extending-creativity#OpenAI\n",
            "https://openai.com/blog/dall-e-2-extending-creativity#content\n",
            "https://openai.com/dall-e-2#content\n",
            "https://openai.com/blog/reducing-bias-and-improving-safety-in-dall-e-2\n",
            "https://openai.com/blog/reducing-bias-and-improving-safety-in-dall-e-2#content\n",
            "https://openai.com/blog/language-model-safety-and-misuse\n",
            "https://openai.com/blog/language-model-safety-and-misuse/#risk-impact-measurement\n",
            "https://openai.com/blog/cooperation-on-safety\n",
            "https://openai.com/research/cooperation-on-safety#content\n",
            "https://openai.com/research?authors=steven-adler\n",
            "https://openai.com/research?authors=tyna-eloundou\n",
            "https://openai.com/research?authors=katie-mayer\n",
            "https://openai.com/blog/economic-impacts\n",
            "https://openai.com/blog?authors=sam-manning\n",
            "https://openai.com/blog/economic-impacts#PamelaMishkin\n",
            "https://openai.com/blog/economic-impacts#content\n",
            "https://openai.com/blog/economic-impacts#TynaEloundou\n",
            "https://openai.com/blog?authors=pamela-mishkin\n",
            "https://openai.com/blog/economic-impacts#SamManning\n",
            "https://openai.com/blog/language-model-safety-and-misuse/#misuse\n",
            "https://openai.com/alignment\n",
            "https://openai.com/research/language-model-safety-and-misuse\n",
            "https://openai.com/research/cooperation-on-safety\n",
            "https://openai.com/safety#content\n",
            "https://openai.com/blog/our-approach-to-alignment-research\n",
            "https://openai.com/blog/our-approach-to-alignment-research#JohnSchulman\n",
            "https://openai.com/blog/our-approach-to-alignment-research#content\n",
            "https://openai.com/blog?authors=john-schulman-2\n",
            "https://openai.com/blog/our-approach-to-alignment-research#JeffreyWu\n",
            "https://openai.com/blog?authors=jeffrey-wu\n",
            "https://openai.com/blog/our-approach-to-alignment-research#JanLeike\n",
            "https://openai.com/blog/openai-codex\n",
            "https://openai.com/blog?authors=wojciech-zaremba\n",
            "https://openai.com/blog/openai-codex#content\n",
            "https://openai.com/blog/openai-codex#GregBrockman\n",
            "https://openai.com/blog/openai-codex#OpenAI\n",
            "https://openai.com/blog/openai-codex#WojciechZaremba\n",
            "https://openai.com/blog/instruction-following/#limitations\n",
            "https://openai.com/research/language-model-safety-and-misuse#content\n",
            "https://openai.com/blog/reducing-bias-and-improving-safety-in-dall-e-2#OpenAI\n",
            "https://openai.com/blog/dall-e-now-available-without-waitlist\n",
            "https://openai.com/blog/dall-e-now-available-in-beta\n",
            "https://openai.com/blog/dall-e-now-available-in-beta#OpenAI\n",
            "https://openai.com/blog/dall-e-now-available-in-beta#content\n",
            "https://openai.com/blog/dall-e-now-available-without-waitlist#content\n",
            "https://openai.com/blog/dall-e-2-update\n",
            "https://openai.com/blog/dall-e-2-update#content\n",
            "https://openai.com/blog/dall-e-2-update#JoanneJang\n",
            "https://openai.com/blog/dall-e-2-update#OpenAI\n",
            "https://openai.com/blog/dall-e\n",
            "https://openai.com/blog/dall-e-now-available-without-waitlist#OpenAI\n",
            "https://openai.com/blog/dall-e-api-now-available-in-public-beta\n",
            "https://openai.com/blog/dall-e-api-now-available-in-public-beta#content\n",
            "https://openai.com/blog/clip\n",
            "https://openai.com/research?authors=alex-nichol\n",
            "https://openai.com/research/a-hazard-analysis-framework-for-code-synthesis-large-language-models\n",
            "https://openai.com/research?models=openai-codex\n",
            "https://openai.com/research?authors=heidy-khlaaf\n",
            "https://openai.com/research?authors=joshua-achiam\n",
            "https://openai.com/research/a-hazard-analysis-framework-for-code-synthesis-large-language-models#content\n",
            "https://openai.com/research?topics=human-feedback\n",
            "https://openai.com/research?topics=computer-vision\n",
            "https://openai.com/research?topics=language\n",
            "https://openai.com/research/measuring-goodharts-law\n",
            "https://openai.com/blog/learning-to-summarize-with-human-feedback/#optimizingtherewardmodel\n",
            "https://openai.com/research/measuring-goodharts-law#content\n",
            "https://openai.com/careers/research-engineer\n",
            "https://openai.com/careers/research-engineer#content\n",
            "https://openai.com/research?topics=generative-models\n",
            "https://openai.com/research?topics=compute\n",
            "https://openai.com/research/efficient-training-of-language-models-to-fill-in-the-middle\n",
            "https://openai.com/research/efficient-training-of-language-models-to-fill-in-the-middle#content\n",
            "https://openai.com/research?authors=mohammad-bavarian\n",
            "https://openai.com/research?authors=christine-mcleavey-payne\n",
            "https://openai.com/research?authors=heewoo-jun\n",
            "https://openai.com/research/techniques-for-training-large-neural-networks\n",
            "https://openai.com/research/techniques-for-training-large-neural-networks#content\n",
            "https://openai.com/research?topics=games\n",
            "https://openai.com/research/critiques\n",
            "https://openai.com/research?topics=reinforcement-learning\n",
            "https://openai.com/research/whisper\n",
            "https://openai.com/research/forecasting-misuse\n",
            "https://openai.com/research/vpt\n",
            "https://openai.com/research?authors=peter-zhokhov\n",
            "https://openai.com/research?authors=joost-huizinga\n",
            "https://openai.com/research?authors=raul-sampedro\n",
            "https://openai.com/research?authors=jeff-clune\n",
            "https://openai.com/research/vpt#content\n",
            "https://openai.com/research?topics=summarization\n",
            "https://openai.com/research?topics=community\n",
            "https://openai.com/research?topics=representation-learning\n",
            "https://openai.com/research?topics=reasoning\n",
            "https://openai.com/research?topics=responsible-ai\n",
            "https://openai.com/research?topics=safety-alignment\n",
            "https://openai.com/research/gpt-4\n",
            "https://openai.com/contributions/gpt-4\n",
            "https://openai.com/research/gpt-4#content\n",
            "https://openai.com/form/researcher-access-program\n",
            "https://openai.com/waitlist/gpt-4-api\n",
            "https://openai.com/en\n",
            "HTTP Error 404: Not Found\n",
            "https://openai.com/blog/how-should-ai-systems-behave\n",
            "https://openai.com/blog/how-should-ai-systems-behave#OpenAI\n",
            "https://openai.com/careers/#open\n",
            "https://openai.com/blog/how-should-ai-systems-behave#content\n",
            "https://openai.com/waitlist/gpt-4\n",
            "https://openai.com/blog/chatgpt\n",
            "https://openai.com/blog/chatgpt#content\n",
            "https://openai.com/blog/chatgpt-plus\n",
            "https://openai.com/blog/chatgpt-plus#content\n",
            "https://openai.com/blog/chatgpt-plus#OpenAI\n",
            "https://openai.com/blog/chatgpt#OpenAI\n",
            "https://openai.com/research/instruction-following\n",
            "https://openai.com/research/teaching-models-to-express-their-uncertainty-in-words\n",
            "https://openai.com/research?authors=stephanie-lin\n",
            "https://openai.com/research?authors=owain-evans\n",
            "https://openai.com/research/teaching-models-to-express-their-uncertainty-in-words#content\n",
            "https://openai.com/research?topics=software-engineering\n",
            "https://openai.com/research?topics=research\n",
            "https://openai.com/research#content\n",
            "https://openai.com/research/hierarchical-text-conditional-image-generation-with-clip-latents\n",
            "https://openai.com/research/hierarchical-text-conditional-image-generation-with-clip-latents#content\n",
            "https://openai.com/research?authors=casey-chu\n",
            "https://openai.com/research/point-e\n",
            "https://openai.com/research?models=point-e\n",
            "https://openai.com/research/point-e#content\n",
            "https://openai.com/research/evolution-through-large-models\n",
            "https://openai.com/research?authors=kenneth-o-stanley\n",
            "https://openai.com/research/evolution-through-large-models#content\n",
            "https://openai.com/research?authors=cathy-yeh\n",
            "https://openai.com/research?authors=jonathan-gordon\n",
            "https://openai.com/research?authors=shawn-jain\n",
            "https://openai.com/research?authors=joel-lehman\n",
            "https://openai.com/research?authors=kamal-ndousse\n",
            "https://openai.com/research?contentTypes=release\n",
            "https://openai.com/research?topics=speech-recognition\n",
            "https://openai.com/research?contentTypes=publication\n",
            "https://openai.com/research?topics=transformers\n",
            "https://openai.com/research/whisper#content\n",
            "https://openai.com/research?contentTypes=milestone\n",
            "https://openai.com/research?topics=open-source\n",
            "https://openai.com/contact-sales\n",
            "https://openai.com/contact-sales#content\n",
            "https://openai.com/blog/introducing-chatgpt-and-whisper-apis#LoganKilpatrick\n",
            "https://openai.com/blog?authors=rachel-lim\n",
            "https://openai.com/blog/introducing-chatgpt-and-whisper-apis#ElieGeorges\n",
            "https://openai.com/blog/announcing-openai-devday#content\n",
            "https://openai.com/blog/moving-ai-governance-forward\n",
            "https://openai.com/blog?topics=responsible-ai\n",
            "https://openai.com/blog/moving-ai-governance-forward#OpenAI\n",
            "https://openai.com/blog/moving-ai-governance-forward#content\n",
            "https://openai.com/blog/insights-from-global-conversations\n",
            "https://openai.com/research/improving-verifiability\n",
            "https://openai.com/blog/improving-verifiability/#authors\n",
            "https://openai.com/research/improving-verifiability#content\n",
            "https://openai.com/blog/our-approach-to-ai-safety\n",
            "https://openai.com/blog/our-approach-to-ai-safety#OpenAI\n",
            "https://openai.com/blog/our-approach-to-ai-safety#content\n",
            "https://openai.com/blog/governance-of-superintelligence?trk=public_post_comment-text\n",
            "https://openai.com/blog/governance-of-superintelligence#IlyaSutskever\n",
            "https://openai.com/blog/governance-of-superintelligence#SamAltman\n",
            "https://openai.com/blog/governance-of-superintelligence#GregBrockman\n",
            "https://openai.com/blog/governance-of-superintelligence#content\n",
            "https://openai.com/blog/insights-from-global-conversations#content\n",
            "https://openai.com/blog/insights-from-global-conversations#OpenAI\n",
            "https://openai.com/form/for-good-use-cases\n",
            "https://openai.com/blog/new-ways-to-manage-your-data-in-chatgpt\n",
            "https://openai.com/blog/new-ways-to-manage-your-data-in-chatgpt#OpenAI\n",
            "https://openai.com/blog/new-ways-to-manage-your-data-in-chatgpt#content\n",
            "https://openai.com/blog/openai-acquires-global-illumination\n",
            "https://openai.com/blog/openai-acquires-global-illumination#OpenAI\n",
            "https://openai.com/blog/openai-acquires-global-illumination#content\n",
            "https://openai.com/blog/teaching-with-ai\n",
            "https://openai.com/blog/teaching-with-ai#OpenAI\n",
            "https://openai.com/blog/teaching-with-ai#content\n",
            "https://openai.com/blog/openai-cybersecurity-grant-program\n",
            "https://openai.com/blog?authors=joel-parish\n",
            "https://openai.com/blog/openai-cybersecurity-grant-program#HaroldNguyen\n",
            "https://openai.com/blog/openai-cybersecurity-grant-program#content\n",
            "https://openai.com/blog?authors=gabriel-bernadett-shapiro\n",
            "https://openai.com/blog/openai-cybersecurity-grant-program#JoelParish\n",
            "https://openai.com/blog?authors=girish-sastry\n",
            "https://openai.com/blog/openai-cybersecurity-grant-program#GirishSastry\n",
            "https://openai.com/blog?authors=bob-rotsted\n",
            "https://openai.com/blog?authors=harold-nguyen\n",
            "https://openai.com/blog/openai-cybersecurity-grant-program#GabrielBernadett-Shapiro\n",
            "https://openai.com/blog/openai-cybersecurity-grant-program#BobRotsted\n",
            "https://openai.com/form/cybersecurity-grant-program\n",
            "https://openai.com/blog/gpt-3-5-turbo-fine-tuning-and-api-updates\n",
            "https://openai.com/api-data-privacy\n",
            "https://openai.com/blog?authors=steven-heidel\n",
            "https://openai.com/blog/gpt-3-5-turbo-fine-tuning-and-api-updates#MichaelWu\n",
            "https://openai.com/blog?authors=andrew-peng\n",
            "https://openai.com/blog/gpt-3-5-turbo-fine-tuning-and-api-updates#content\n",
            "https://openai.com/blog/gpt-3-5-turbo-fine-tuning-and-api-updates#JohnAllard\n",
            "https://openai.com/blog/gpt-3-5-turbo-fine-tuning-and-api-updates#LoganKilpatrick\n",
            "https://openai.com/blog/gpt-3-5-turbo-fine-tuning-and-api-updates#StevenHeidel\n",
            "https://openai.com/blog/gpt-3-5-turbo-fine-tuning-and-api-updates#AndrewPeng\n",
            "https://openai.com/blog?authors=john-allard\n",
            "https://openai.com/blog/introducing-openai-dublin\n",
            "https://openai.com/blog/introducing-openai-dublin#content\n",
            "https://openai.com/blog/introducing-openai-dublin#OpenAI\n",
            "https://openai.com/blog/introducing-superalignment\n",
            "https://openai.com/blog/introducing-superalignment#content\n",
            "https://openai.com/blog/introducing-superalignment#IlyaSutskever\n",
            "https://openai.com/careers/research-engineer-superalignment\n",
            "https://openai.com/mission\n",
            "HTTP Error 404: Not Found\n",
            "https://openai.com/careers/research-engineer-superalignment#content\n",
            "https://openai.com/blog/governance-of-superintelligence\n",
            "https://openai.com/careers/research-scientist-superalignment\n",
            "https://openai.com/careers/research-scientist-superalignment#content\n",
            "https://openai.com/careers/research-manager-superalignment\n",
            "https://openai.com/careers/research-manager-superalignment#content\n",
            "https://openai.com/blog/introducing-superalignment#JanLeike\n",
            "https://openai.com/blog/introducing-chatgpt-enterprise\n",
            "https://openai.com/blog/introducing-chatgpt-enterprise#content\n",
            "https://openai.com/blog/chatgpt-plugins\n",
            "https://openai.com/bot\n",
            "Unable to parse page https://openai.com/bot due to JavaScript being required\n",
            "HTTP Error 403: Forbidden\n",
            "https://openai.com/waitlist/plugins\n",
            "https://openai.com/research/webgpt\n",
            "https://openai.com/blog/chatgpt-plugins#content\n",
            "https://openai.com/blog/chatgpt-plugins#OpenAI\n",
            "http://openai.com/waitlist/plugins\n",
            "https://openai.com/blog/planning-for-agi-and-beyond\n",
            "https://openai.com/blog/planning-for-agi-and-beyond#content\n",
            "https://openai.com/blog/planning-for-agi-and-beyond#SamAltman\n",
            "https://openai.com/blog/introducing-chatgpt-enterprise#OpenAI\n",
            "https://openai.com/blog/gpt-4-api-general-availability\n",
            "https://openai.com/blog/gpt-4-api-general-availability#OpenAI\n",
            "https://openai.com/blog/gpt-4-api-general-availability#content\n",
            "https://openai.com/blog#content\n",
            "https://openai.com/blog/function-calling-and-other-api-updates\n",
            "https://openai.com/blog/introducing-chatgpt-and-whisper-apis#:~:text=Chat%20guide.-,ChatGPT%20upgrades,-We%20are%20constantly\n",
            "https://openai.com/blog/function-calling-and-other-api-updates#content\n",
            "https://openai.com/blog?authors=jeff-harris\n",
            "https://openai.com/blog/function-calling-and-other-api-updates#JeffHarris\n",
            "https://openai.com/blog/function-calling-and-other-api-updates#AttyEleti\n",
            "https://openai.com/blog/function-calling-and-other-api-updates#LoganKilpatrick\n",
            "https://openai.com/blog/custom-instructions-for-chatgpt\n",
            "https://openai.com/blog/custom-instructions-for-chatgpt#OpenAI\n",
            "https://openai.com/blog/custom-instructions-for-chatgpt#content\n",
            "https://openai.com/blog/red-teaming-network\n",
            "https://openai.com/form/red-teaming-network\n",
            "https://openai.com/blog/red-teaming-network#content\n",
            "https://openai.com/blog/red-teaming-network#OpenAI\n",
            "https://openai.com/blog/democratic-inputs-to-ai\n",
            "https://openai.com/blog/democratic-inputs-to-ai#ShibaniSanturkar\n",
            "https://openai.com/blog/democratic-inputs-to-ai#SandhiniAgarwal\n",
            "https://openai.com/blog?authors=arka-dhar\n",
            "https://openai.com/blog/democratic-inputs-to-ai#content\n",
            "https://openai.com/blog/democratic-inputs-to-ai#TynaEloundou\n",
            "https://openai.com/blog/democratic-inputs-to-ai#JadeLeung\n",
            "https://openai.com/blog?authors=shibani-santurkar\n",
            "https://openai.com/blog/democratic-inputs-to-ai#LamaAhmad\n",
            "https://openai.com/blog?authors=jade-leung\n",
            "https://openai.com/blog/democratic-inputs-to-ai#ArkaDhar\n",
            "https://openai.com/blog/democratic-inputs-to-ai#WojciechZaremba\n",
            "https://openai.com/blog/frontier-model-forum\n",
            "https://openai.com/blog/frontier-model-forum#content\n",
            "https://openai.com/blog/frontier-model-forum#OpenAI\n",
            "https://openai.com/blog/openai-partners-with-scale-to-provide-support-for-enterprises-fine-tuning-models\n",
            "https://openai.com/blog/openai-partners-with-scale-to-provide-support-for-enterprises-fine-tuning-models#OpenAI\n",
            "https://openai.com/blog/openai-partners-with-scale-to-provide-support-for-enterprises-fine-tuning-models#content\n",
            "https://openai.com/blog/introducing-openai-london\n",
            "https://openai.com/blog/introducing-openai-london#OpenAI\n",
            "https://openai.com/blog/introducing-openai-london#content\n",
            "https://openai.com/blog/partnership-with-american-journalism-project-to-support-local-news\n",
            "https://openai.com/blog/partnership-with-american-journalism-project-to-support-local-news#content\n",
            "https://openai.com/blog/using-gpt-4-for-content-moderation\n",
            "https://openai.com/blog/using-gpt-4-for-content-moderation#VikGoel\n",
            "https://openai.com/blog?authors=vik-goel\n",
            "https://openai.com/blog?authors=andrea-vallone\n",
            "https://openai.com/blog/using-gpt-4-for-content-moderation#LilianWeng\n",
            "https://openai.com/blog/using-gpt-4-for-content-moderation#AndreaVallone\n",
            "https://openai.com/blog/using-gpt-4-for-content-moderation#content\n",
            "https://openai.com/blog/bug-bounty-program#content\n",
            "https://openai.com/blog/bug-bounty-program#OpenAI\n",
            "https://openai.com/careers/search?c=security\n",
            "https://openai.com/careers/transportation-coordinator-contingent\n",
            "https://openai.com/careers/transportation-coordinator-contingent#content\n",
            "https://openai.com/careers/software-engineer-detection-engineering\n",
            "https://openai.com/careers/software-engineer-detection-engineering#content\n",
            "https://openai.com/careers/kernel-engineer\n",
            "https://openai.com/careers/kernel-engineer#content\n",
            "https://openai.com/careers/hr-business-partner-go-to-market-and-sales\n",
            "https://openai.com/careers/hr-business-partner-go-to-market-and-sales#content\n",
            "https://openai.com/careers/recruiting-coordinator-contingent\n",
            "https://openai.com/careers/recruiting-coordinator-contingent#content\n",
            "https://openai.com/careers/software-engineer-infrastructure\n",
            "https://openai.com/careers/software-engineer-infrastructure#content\n",
            "https://openai.com/blog/scaling-kubernetes-to-7500-nodes\n",
            "https://openai.com/research?authors=benjamin-chess\n",
            "https://openai.com/research/scaling-kubernetes-to-7500-nodes#content\n",
            "https://openai.com/blog/scaling-kubernetes-to-2500-nodes\n",
            "https://openai.com/research/scaling-kubernetes-to-2500-nodes#content\n",
            "https://openai.com/blog/scaling-kubernetes-to-7500-nodes/#unsolvedproblems\n",
            "https://openai.com/careers/media-relations-corporate-communications\n",
            "https://openai.com/careers/media-relations-corporate-communications#content\n",
            "https://openai.com/careers/front-end-engineer-review-training-platform\n",
            "https://openai.com/careers/front-end-engineer-review-training-platform#content\n",
            "https://openai.com/careers/backend-engineer-review-training-platform-3\n",
            "https://openai.com/careers/backend-engineer-review-training-platform-3#content\n",
            "https://openai.com/careers/international-payroll-specialist\n",
            "https://openai.com/careers/international-payroll-specialist#content\n",
            "https://openai.com/careers/account-associate-2\n",
            "https://openai.com/careers/account-associate-2#content\n",
            "https://openai.com/careers/distributed-systemsml-engineer\n",
            "https://openai.com/careers/distributed-systemsml-engineer#content\n",
            "https://openai.com/careers/manager-scientist-deployment\n",
            "https://openai.com/careers/manager-scientist-deployment#content\n",
            "https://openai.com/careers/international-policy-analyst-global-affairs\n",
            "https://openai.com/careers/international-policy-analyst-global-affairs#content\n",
            "https://openai.com/careers/software-engineer-engineering-acceleration\n",
            "https://openai.com/careers/software-engineer-engineering-acceleration#content\n",
            "https://openai.com/careers/associate-general-counsel-emea-2\n",
            "https://openai.com/careers/associate-general-counsel-emea-2#content\n",
            "https://openai.com/careers/us-congressional-lead-global-affairs\n",
            "https://openai.com/careers/us-congressional-lead-global-affairs#content\n",
            "https://openai.com/careers/software-engineer-safety\n",
            "https://openai.com/careers/software-engineer-safety#content\n",
            "https://openai.com/careers/search#content\n",
            "https://openai.com/careers/investigations-and-compliance-counsel\n",
            "https://openai.com/careers/investigations-and-compliance-counsel#content\n",
            "https://openai.com/careers/account-associate\n",
            "https://openai.com/careers/account-associate#content\n",
            "https://openai.com/careers/engineering-manager-enterprise-api\n",
            "https://openai.com/careers/engineering-manager-enterprise-api#content\n",
            "https://openai.com/careers/security-engineer-partnerships\n",
            "https://openai.com/careers/security-engineer-partnerships#content\n",
            "https://openai.com/careers/software-engineer-model-inference\n",
            "https://openai.com/careers/software-engineer-model-inference#content\n",
            "https://openai.com/careers/mac-workplace-coordinator\n",
            "https://openai.com/careers/mac-workplace-coordinator#content\n",
            "https://openai.com/careers/account-director\n",
            "https://openai.com/careers/account-director#content\n",
            "https://openai.com/careers/senior-revenue-accountant-deal-desk\n",
            "https://openai.com/careers/senior-revenue-accountant-deal-desk#content\n",
            "https://openai.com/blog/api-no-waitlist\n",
            "https://openai.com/blog/api-no-waitlist#OpenAI\n",
            "https://openai.com/blog/api-no-waitlist#content\n",
            "https://openai.com/blog/codex-apps\n",
            "https://openai.com/blog/codex-apps#OpenAI\n",
            "https://openai.com/blog/codex-apps#content\n",
            "https://openai.com/careers/front-end-engineer-review-training-platform-2\n",
            "https://openai.com/careers/front-end-engineer-review-training-platform-2#content\n",
            "https://openai.com/careers/privacy-program-manager\n",
            "https://openai.com/careers/privacy-program-manager#content\n",
            "https://openai.com/careers/senior-software-engineer-identity-platform\n",
            "https://openai.com/careers/senior-software-engineer-identity-platform#content\n",
            "https://openai.com/careers/compliance-operations-analyst\n",
            "https://openai.com/careers/compliance-operations-analyst#content\n",
            "https://openai.com/careers/software-engineer-triton-compiler\n",
            "https://openai.com/careers/software-engineer-triton-compiler#content\n",
            "https://openai.com/careers/back-end-engineer-core-data-systems\n",
            "https://openai.com/careers/back-end-engineer-core-data-systems#content\n",
            "https://openai.com/careers/graph-compiler-engineer\n",
            "https://openai.com/careers/graph-compiler-engineer#content\n",
            "https://openai.com/careers/engineering-manager-chatgpt-growth\n",
            "https://openai.com/careers/engineering-manager-chatgpt-growth#content\n",
            "https://openai.com/careers/software-engineer-backend\n",
            "https://openai.com/careers/software-engineer-backend#content\n",
            "https://openai.com/careers/front-end-engineer-evals-understanding\n",
            "https://openai.com/careers/front-end-engineer-evals-understanding#content\n",
            "https://openai.com/careers/full-stack-engineer-3rd-party-data\n",
            "https://openai.com/careers/full-stack-engineer-3rd-party-data#content\n",
            "https://openai.com/careers/software-engineer-leverage-engineering\n",
            "https://openai.com/careers/software-engineer-leverage-engineering#content\n",
            "https://openai.com/careers/engineering-manager-fine-tuning-api\n",
            "https://openai.com/careers/engineering-manager-fine-tuning-api#content\n",
            "https://openai.com/careers/machine-learning-engineer-moderation\n",
            "https://openai.com/careers/machine-learning-engineer-moderation#content\n",
            "https://openai.com/careers/assistant-controller\n",
            "https://openai.com/careers/assistant-controller#content\n",
            "https://openai.com/careers/senior-corporate-counsel\n",
            "https://openai.com/careers/senior-corporate-counsel#content\n",
            "https://openai.com/careers/backend-engineer-core-data-systems\n",
            "https://openai.com/careers/backend-engineer-core-data-systems#content\n",
            "https://openai.com/careers/full-stack-developer-communications-design\n",
            "https://openai.com/careers/full-stack-developer-communications-design#content\n",
            "https://openai.com/careers/dl-hwsw-codesign-engineer\n",
            "https://openai.com/careers/dl-hwsw-codesign-engineer#content\n",
            "https://openai.com/careers/front-end-engineer-evals-understanding-2\n",
            "https://openai.com/careers/front-end-engineer-evals-understanding-2#content\n",
            "https://openai.com/careers/research-scientist-machine-learning\n",
            "https://openai.com/careers/research-scientist-machine-learning#content\n",
            "https://openai.com/careers/compliance-engineer\n",
            "https://openai.com/careers/compliance-engineer#content\n",
            "https://openai.com/careers/software-engineer-developer-productivity\n",
            "https://openai.com/careers/software-engineer-developer-productivity#content\n",
            "https://openai.com/careers/software-engineer-full-stack-dalle\n",
            "https://openai.com/careers/software-engineer-full-stack-dalle#content\n",
            "https://openai.com/careers/solutions-architect-london\n",
            "https://openai.com/careers/solutions-architect-london#content\n",
            "https://openai.com/careers/commercial-counsel\n",
            "https://openai.com/careers/commercial-counsel#content\n",
            "https://openai.com/careers/eu-policy-and-partnerships-lead\n",
            "https://openai.com/careers/eu-policy-and-partnerships-lead#content\n",
            "https://openai.com/careers/software-engineer-developer-experience\n",
            "https://openai.com/careers/software-engineer-developer-experience#content\n",
            "https://openai.com/careers/engineering-manager-ai-inference-systems\n",
            "https://openai.com/careers/engineering-manager-ai-inference-systems#content\n",
            "https://openai.com/careers/security-engineer-detection-and-response\n",
            "https://openai.com/careers/security-engineer-detection-and-response#content\n",
            "https://openai.com/careers/security-engineer-detection-response\n",
            "https://openai.com/careers/security-engineer-detection-response#content\n",
            "https://openai.com/careers/ml-user-experience-engineer\n",
            "https://openai.com/careers/ml-user-experience-engineer#content\n",
            "https://openai.com/careers/senior-ip-counsel\n",
            "https://openai.com/careers/senior-ip-counsel#content\n",
            "https://openai.com/careers/account-engineer\n",
            "https://openai.com/careers/account-engineer#content\n",
            "https://openai.com/careers/data-infrastructure-engineer\n",
            "https://openai.com/careers/data-infrastructure-engineer#content\n",
            "https://openai.com/careers/backend-engineer-review-training-platform\n",
            "https://openai.com/careers/backend-engineer-review-training-platform#content\n",
            "https://openai.com/careers/solutions-architect\n",
            "https://openai.com/careers/solutions-architect#content\n",
            "https://openai.com/careers/backend-engineer-evals-understanding\n",
            "https://openai.com/careers/backend-engineer-evals-understanding#content\n",
            "https://openai.com/careers/software-engineer-infrastructure-2\n",
            "https://openai.com/careers/software-engineer-infrastructure-2#content\n",
            "https://openai.com/careers/strategic-finance-infrastructure\n",
            "https://openai.com/careers/strategic-finance-infrastructure#content\n",
            "https://openai.com/careers/backend-engineer-review-training-platform-2\n",
            "https://openai.com/careers/backend-engineer-review-training-platform-2#content\n",
            "https://openai.com/careers/backend-engineer-evals-understanding-2\n",
            "https://openai.com/careers/backend-engineer-evals-understanding-2#content\n",
            "https://openai.com/careers/software-engineer-distributed-systems\n",
            "https://openai.com/careers/software-engineer-distributed-systems#content\n",
            "https://openai.com/careers/research-scientist\n",
            "https://openai.com/careers/research-scientist#content\n",
            "https://openai.com/careers/hwsw-co-design-engineer\n",
            "https://openai.com/careers/hwsw-co-design-engineer#content\n",
            "https://openai.com/careers/research-scientist-safety\n",
            "https://openai.com/careers/research-scientist-safety#content\n",
            "https://openai.com/careers/associate-general-counsel-commercial-and-go-to-market\n",
            "https://openai.com/careers/associate-general-counsel-commercial-and-go-to-market#content\n",
            "https://openai.com/careers/software-engineer-model-inference-2\n",
            "https://openai.com/careers/software-engineer-model-inference-2#content\n",
            "https://openai.com/careers/software-engineer-hardware-health\n",
            "https://openai.com/careers/software-engineer-hardware-health#content\n",
            "https://openai.com/careers/data-scientist-product\n",
            "https://openai.com/careers/data-scientist-product#content\n",
            "https://openai.com/careers/software-engineer-backend-infra\n",
            "https://openai.com/careers/software-engineer-backend-infra#content\n",
            "https://openai.com/careers/media-relations-europe-lead\n",
            "https://openai.com/careers/media-relations-europe-lead#content\n",
            "https://openai.com/careers/software-engineer-anti-fraud-abuse\n",
            "https://openai.com/careers/software-engineer-anti-fraud-abuse#content\n",
            "https://openai.com/careers/ireland-policy-and-partnerships-lead-global-affairs\n",
            "https://openai.com/careers/ireland-policy-and-partnerships-lead-global-affairs#content\n",
            "https://openai.com/careers/software-engineer-privacy-2\n",
            "https://openai.com/careers/software-engineer-privacy-2#content\n",
            "https://openai.com/careers/software-engineer-fullstack\n",
            "https://openai.com/careers/software-engineer-fullstack#content\n",
            "https://openai.com/careers/associate-general-counsel-emea\n",
            "https://openai.com/careers/associate-general-counsel-emea#content\n",
            "https://openai.com/careers/software-engineer-chatgpt-data-flywheel\n",
            "https://openai.com/careers/software-engineer-chatgpt-data-flywheel#content\n",
            "https://openai.com/careers/account-engineer-london\n",
            "https://openai.com/careers/account-engineer-london#content\n",
            "https://openai.com/careers/customer-success-manager\n",
            "https://openai.com/careers/customer-success-manager#content\n",
            "https://openai.com/blog?topics=announcements\n",
            "https://openai.com/careers/research-engineer-privacy\n",
            "https://openai.com/careers/research-engineer-privacy#content\n",
            "https://openai.com/careers/search\n",
            "https://openai.com/research\n",
            "https://openai.com/policies\n",
            "https://openai.com/policies/plugin-terms\n",
            "https://openai.com/policies/terms-of-use\n",
            "https://openai.com/policies/service-terms\n",
            "https://openai.com/policies/data-processing-addendum\n",
            "https://openai.com/policies#content\n",
            "https://openai.com/policies/sharing-publication-policy\n",
            "https://openai.com/policies/service-credit-terms\n",
            "https://openai.com/policies/coordinated-vulnerability-disclosure-policy\n",
            "https://openai.com/about\n",
            "https://openai.com/about#content\n",
            "https://openai.com/our-structure\n",
            "https://openai.com/blog/introducing-openai\n",
            "https://openai.com/blog/introducing-openai#OpenAI\n",
            "https://openai.com/blog/introducing-openai#IlyaSutskever\n",
            "https://openai.com/blog/introducing-openai#content\n",
            "https://openai.com/blog/introducing-openai#GregBrockman\n",
            "https://openai.com/blog/microsoft-invests-in-and-partners-with-openai\n",
            "https://openai.com/blog/microsoft-invests-in-and-partners-with-openai#GregBrockman\n",
            "https://openai.com/blog/microsoft-invests-in-and-partners-with-openai#content\n",
            "https://openai.com/blog/openai-and-microsoft-extend-partnership\n",
            "https://openai.com/blog/openai-and-microsoft-extend-partnership#content\n",
            "https://openai.com/blog/openai-scholars-2021-final-projects\n",
            "https://openai.com/blog/openai-scholars-2021-final-projects#OpenAI\n",
            "https://openai.com/scholars\n",
            "HTTP Error 404: Not Found\n",
            "https://openai.com/blog/openai-scholars-2021-final-projects#content\n",
            "https://openai.com/our-structure#content\n",
            "https://openai.com/blog/new-and-improved-embedding-model\n",
            "https://openai.com/blog?authors=ryan-greene\n",
            "https://openai.com/blog/new-and-improved-embedding-model#LilianWeng\n",
            "https://openai.com/blog?authors=arvind-neelakantan\n",
            "https://openai.com/blog/new-and-improved-embedding-model#TedSanders\n",
            "https://openai.com/blog/new-and-improved-embedding-model#ArvindNeelakantan\n",
            "https://openai.com/blog/new-and-improved-embedding-model#content\n",
            "https://openai.com/blog?authors=ted-sanders\n",
            "https://openai.com/blog/new-and-improved-embedding-model#RyanGreene\n",
            "https://openai.com/blog/introducing-text-and-code-embeddings\n",
            "https://openai.com/blog/introducing-text-and-code-embeddings#JoanneJang\n",
            "https://openai.com/blog/introducing-text-and-code-embeddings#BorisPower\n",
            "https://openai.com/blog/introducing-text-and-code-embeddings#ArvindNeelakantan\n",
            "https://openai.com/blog?authors=boris-power\n",
            "https://openai.com/blog/introducing-text-and-code-embeddings#content\n",
            "IncompleteRead(95999 bytes read, 21174 more expected)\n",
            "https://openai.com/blog/introducing-text-and-code-embeddings#LilianWeng\n",
            "https://openai.com/research/language-models-can-explain-neurons-in-language-models\n",
            "https://openai.com/research/language-models-can-explain-neurons-in-language-models#content\n",
            "https://openai.com/research?authors=steven-bills\n",
            "https://openai.com/research?authors=daniel-mossing\n",
            "https://openai.com/research/microscope\n",
            "https://openai.com/research?authors=nick-cammarata\n",
            "https://openai.com/research/microscope#content\n",
            "https://openai.com/research?authors=chris-olah\n",
            "https://openai.com/research?authors=ludwig-schubert\n",
            "https://openai.com/research?authors=shan-carter\n",
            "https://openai.com/research?topics=interpretability\n",
            "https://openai.com/research?authors=henk-tillman\n",
            "https://openai.com/safety\n",
            "https://openai.com/research/improving-mathematical-reasoning-with-process-supervision\n",
            "https://openai.com/research/improving-mathematical-reasoning-with-process-supervision#content\n",
            "https://openai.com/research?authors=yura-burda\n",
            "https://openai.com/research?authors=harri-edwards\n",
            "https://openai.com/research?authors=hunter-lightman\n",
            "https://openai.com/research/improving-mathematical-reasoning-with-process-supervision#samples\n",
            "https://openai.com/product\n",
            "https://openai.com/enterprise\n",
            "https://openai.com/enterprise#content\n",
            "https://openai.com/brand\n",
            "https://openai.com/brand#content\n",
            "https://openai.com/careers\n",
            "https://openai.com/chatgpt\n",
            "https://openai.com/blog/introducing-the-chatgpt-app-for-ios\n",
            "https://openai.com/product/gpt-4\n",
            "https://openai.com/gpt-4#content\n",
            "https://openai.com/blog/introducing-the-chatgpt-app-for-ios#content\n",
            "https://openai.com/chatgpt#content\n",
            "https://openai.com/research/confidence-building-measures-for-artificial-intelligence\n",
            "https://openai.com/research/confidence-building-measures-for-artificial-intelligence#content\n",
            "https://openai.com/research/overview\n",
            "https://openai.com/research/overview#content\n",
            "https://openai.com/research/solving-rubiks-cube\n",
            "https://openai.com/research/emergent-tool-use\n",
            "https://openai.com/research/emergent-tool-use#content\n",
            "https://openai.com/blog/competitive-self-play\n",
            "https://openai.com/research?topics=multi-agent\n",
            "https://openai.com/research?authors=ingmar-kanitscheider\n",
            "https://openai.com/research?authors=todor-markov\n",
            "https://openai.com/research?authors=yi-wu\n",
            "https://openai.com/research/summarizing-books\n",
            "https://openai.com/research/musenet\n",
            "https://openai.com/research/image-gpt\n",
            "https://openai.com/research/language-models-are-few-shot-learners\n",
            "https://openai.com/research?authors=ariel-herbert-voss\n",
            "https://openai.com/research?authors=melanie-subbiah\n",
            "https://openai.com/research?authors=nick-ryder\n",
            "https://openai.com/research?authors=tom-henighan\n",
            "https://openai.com/research/language-models-are-few-shot-learners#content\n",
            "https://openai.com/research?authors=arvind-neelakantan\n",
            "https://openai.com/research?authors=clemens-winter\n",
            "https://openai.com/research?authors=sam-mccandlish\n",
            "https://openai.com/research?authors=pranav-shyam\n",
            "https://openai.com/research?authors=jared-kaplan\n",
            "https://openai.com/research?authors=benjamin-mann\n",
            "https://openai.com/research/jukebox\n",
            "https://openai.com/research?models=jukebox\n",
            "https://openai.com/blog/jukebox/#limitations\n",
            "https://openai.com/blog/jukebox/#rf33\n",
            "https://openai.com/research/jukebox#content\n",
            "https://openai.com/five\n",
            "https://openai.com/pricing\n",
            "https://openai.com/pricing#language-models\n",
            "https://openai.com/pricing#content\n",
            "https://openai.com/#content\n",
            "https://openai.com/residency\n",
            "https://openai.com/residency#content\n",
            "https://openai.com/research/solving-math-word-problems\n",
            "https://openai.com/research/economic-impacts\n",
            "https://openai.com/research/economic-impacts#content\n",
            "https://openai.com/research/frontier-ai-regulation\n",
            "https://openai.com/research/frontier-ai-regulation#content\n",
            "https://openai.com/policies/privacy-policy\n",
            "https://openai.com/charter\n",
            "https://openai.com/charter#content\n",
            "https://openai.com/gpt-4\n",
            "https://openai.com/customer-stories\n",
            "https://openai.com/blog\n",
            "https://openai.com\n",
            "https://openai.com/enterprise-privacy\n",
            "https://openai.com/dall-e-3\n",
            "https://openai.com/dall-e-3#content\n"
          ]
        }
      ],
      "source": [
        "import requests\n",
        "import re\n",
        "import urllib.request\n",
        "from bs4 import BeautifulSoup\n",
        "from collections import deque\n",
        "from html.parser import HTMLParser\n",
        "from urllib.parse import urlparse\n",
        "import os\n",
        "\n",
        "# Regex pattern to match a URL\n",
        "HTTP_URL_PATTERN = r'^http[s]*://.+'\n",
        "\n",
        "# Define root domain to crawl\n",
        "domain = \"openai.com\"\n",
        "full_url = \"https://openai.com/\"\n",
        "\n",
        "# Create a class to parse the HTML and get the hyperlinks\n",
        "class HyperlinkParser(HTMLParser):\n",
        "    def __init__(self):\n",
        "        super().__init__()\n",
        "        # Create a list to store the hyperlinks\n",
        "        self.hyperlinks = []\n",
        "\n",
        "    # Override the HTMLParser's handle_starttag method to get the hyperlinks\n",
        "    def handle_starttag(self, tag, attrs):\n",
        "        attrs = dict(attrs)\n",
        "\n",
        "        # If the tag is an anchor tag and it has an href attribute, add the href attribute to the list of hyperlinks\n",
        "        if tag == \"a\" and \"href\" in attrs:\n",
        "            self.hyperlinks.append(attrs[\"href\"])\n",
        "\n",
        "# Function to get the hyperlinks from a URL\n",
        "def get_hyperlinks(url):\n",
        "\n",
        "    # Try to open the URL and read the HTML\n",
        "    try:\n",
        "        # Open the URL and read the HTML\n",
        "        with urllib.request.urlopen(url) as response:\n",
        "\n",
        "            # If the response is not HTML, return an empty list\n",
        "            if not response.info().get('Content-Type').startswith(\"text/html\"):\n",
        "                return []\n",
        "\n",
        "            # Decode the HTML\n",
        "            html = response.read().decode('utf-8')\n",
        "    except Exception as e:\n",
        "        print(e)\n",
        "        return []\n",
        "\n",
        "    # Create the HTML Parser and then Parse the HTML to get hyperlinks\n",
        "    parser = HyperlinkParser()\n",
        "    parser.feed(html)\n",
        "\n",
        "    return parser.hyperlinks\n",
        "\n",
        "# Function to get the hyperlinks from a URL that are within the same domain\n",
        "def get_domain_hyperlinks(local_domain, url):\n",
        "    clean_links = []\n",
        "    for link in set(get_hyperlinks(url)):\n",
        "        clean_link = None\n",
        "\n",
        "        # If the link is a URL, check if it is within the same domain\n",
        "        if re.search(HTTP_URL_PATTERN, link):\n",
        "            # Parse the URL and check if the domain is the same\n",
        "            url_obj = urlparse(link)\n",
        "            if url_obj.netloc == local_domain:\n",
        "                clean_link = link\n",
        "\n",
        "        # If the link is not a URL, check if it is a relative link\n",
        "        else:\n",
        "            if link.startswith(\"/\"):\n",
        "                link = link[1:]\n",
        "            elif link.startswith(\"#\") or link.startswith(\"mailto:\"):\n",
        "                continue\n",
        "            clean_link = \"https://\" + local_domain + \"/\" + link\n",
        "\n",
        "        if clean_link is not None:\n",
        "            if clean_link.endswith(\"/\"):\n",
        "                clean_link = clean_link[:-1]\n",
        "            clean_links.append(clean_link)\n",
        "\n",
        "    # Return the list of hyperlinks that are within the same domain\n",
        "    return list(set(clean_links))\n",
        "\n",
        "\n",
        "def crawl(url):\n",
        "    # Parse the URL and get the domain\n",
        "    local_domain = urlparse(url).netloc\n",
        "\n",
        "    # Create a queue to store the URLs to crawl\n",
        "    queue = deque([url])\n",
        "\n",
        "    # Create a set to store the URLs that have already been seen (no duplicates)\n",
        "    seen = set([url])\n",
        "\n",
        "    # Create a directory to store the text files\n",
        "    if not os.path.exists(\"text/\"):\n",
        "            os.mkdir(\"text/\")\n",
        "\n",
        "    if not os.path.exists(\"text/\"+local_domain+\"/\"):\n",
        "            os.mkdir(\"text/\" + local_domain + \"/\")\n",
        "\n",
        "    # Create a directory to store the csv files\n",
        "    if not os.path.exists(\"processed\"):\n",
        "            os.mkdir(\"processed\")\n",
        "\n",
        "    # While the queue is not empty, continue crawling\n",
        "    while queue:\n",
        "\n",
        "        # Get the next URL from the queue\n",
        "        url = queue.pop()\n",
        "        print(url) # for debugging and to see the progress\n",
        "\n",
        "        # Save text from the url to a <url>.txt file\n",
        "        with open('text/'+local_domain+'/'+url[8:].replace(\"/\", \"_\") + \".txt\", \"w\") as f:\n",
        "\n",
        "            # Get the text from the URL using BeautifulSoup\n",
        "            soup = BeautifulSoup(requests.get(url).text, \"html.parser\")\n",
        "\n",
        "            # Get the text but remove the tags\n",
        "            text = soup.get_text()\n",
        "\n",
        "            # If the crawler gets to a page that requires JavaScript, it will stop the crawl\n",
        "            if (\"You need to enable JavaScript to run this app.\" in text):\n",
        "                print(\"Unable to parse page \" + url + \" due to JavaScript being required\")\n",
        "\n",
        "            # Otherwise, write the text to the file in the text directory\n",
        "            f.write(text)\n",
        "\n",
        "        # Get the hyperlinks from the URL and add them to the queue\n",
        "        for link in get_domain_hyperlinks(local_domain, url):\n",
        "            if link not in seen:\n",
        "                queue.append(link)\n",
        "                seen.add(link)\n",
        "\n",
        "crawl(full_url)"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import os\n",
        "print(len(os.listdir(\"/content/text/openai.com\")))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "HIoSmHGGVg9t",
        "outputId": "361e71db-4645-41a5-cef8-9221240485af"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "1079\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "zvAyD_fOSPbU"
      },
      "outputs": [],
      "source": [
        "def remove_newlines(serie):\n",
        "    serie = serie.str.replace('\\n', ' ')\n",
        "    serie = serie.str.replace('\\\\n', ' ')\n",
        "    serie = serie.str.replace('  ', ' ')\n",
        "    serie = serie.str.replace('  ', ' ')\n",
        "    return serie"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 241
        },
        "id": "QDvrAcKeSPbV",
        "outputId": "d0be6468-47fc-4777-894d-5f41976658ef"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "<ipython-input-3-429202555403>:3: FutureWarning: The default value of regex will change from True to False in a future version.\n",
            "  serie = serie.str.replace('\\\\n', ' ')\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                      fname  \\\n",
              "0                            blog openai lp   \n",
              "1       blog dall e introducing outpainting   \n",
              "2  blog introducing the chatgpt app for ios   \n",
              "3  careers software engineer infrastructure   \n",
              "4      blog openai technical goals#ElonMusk   \n",
              "\n",
              "                                                text  \n",
              "0  blog openai lp.  OpenAI LP    CloseSearch Subm...  \n",
              "1  blog dall e introducing outpainting.  DALL·E: ...  \n",
              "2  blog introducing the chatgpt app for ios.  Int...  \n",
              "3  careers software engineer infrastructure.  Sof...  \n",
              "4  blog openai technical goals#ElonMusk.  OpenAI ...  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-32233833-a338-4626-9e93-59e1ee698b03\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>fname</th>\n",
              "      <th>text</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>blog openai lp</td>\n",
              "      <td>blog openai lp.  OpenAI LP    CloseSearch Subm...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>blog dall e introducing outpainting</td>\n",
              "      <td>blog dall e introducing outpainting.  DALL·E: ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>blog introducing the chatgpt app for ios</td>\n",
              "      <td>blog introducing the chatgpt app for ios.  Int...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>careers software engineer infrastructure</td>\n",
              "      <td>careers software engineer infrastructure.  Sof...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>blog openai technical goals#ElonMusk</td>\n",
              "      <td>blog openai technical goals#ElonMusk.  OpenAI ...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-32233833-a338-4626-9e93-59e1ee698b03')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-32233833-a338-4626-9e93-59e1ee698b03 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-32233833-a338-4626-9e93-59e1ee698b03');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-4fc1718b-0122-4547-abe2-961d03d42a67\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-4fc1718b-0122-4547-abe2-961d03d42a67')\"\n",
              "            title=\"Suggest charts.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-4fc1718b-0122-4547-abe2-961d03d42a67 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "    </div>\n",
              "  </div>\n"
            ]
          },
          "metadata": {},
          "execution_count": 4
        }
      ],
      "source": [
        "import pandas as pd\n",
        "\n",
        "# Create a list to store the text files\n",
        "texts=[]\n",
        "\n",
        "# Get all the text files in the text directory\n",
        "for file in os.listdir(\"text/\" + domain + \"/\"):\n",
        "\n",
        "    # Open the file and read the text\n",
        "    with open(\"text/\" + domain + \"/\" + file, \"r\") as f:\n",
        "        text = f.read()\n",
        "\n",
        "        # Omit the first 11 lines and the last 4 lines, then replace -, _, and #update with spaces.\n",
        "        texts.append((file[11:-4].replace('-',' ').replace('_', ' ').replace('#update',''), text))\n",
        "\n",
        "# Create a dataframe from the list of texts\n",
        "df = pd.DataFrame(texts, columns = ['fname', 'text'])\n",
        "\n",
        "# Set the text column to be the raw text with the newlines removed\n",
        "df['text'] = df.fname + \". \" + remove_newlines(df.text)\n",
        "df.to_csv('processed/scraped.csv')\n",
        "df.head()"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install tiktoken --quiet"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "DIE1JKBgWBcN",
        "outputId": "495459ef-7d22-4e25-bce1-49d789d9fcec"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[?25l     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m0.0/2.0 MB\u001b[0m \u001b[31m?\u001b[0m eta \u001b[36m-:--:--\u001b[0m\r\u001b[2K     \u001b[91m━━━\u001b[0m\u001b[90m╺\u001b[0m\u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m0.2/2.0 MB\u001b[0m \u001b[31m4.4 MB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m\r\u001b[2K     \u001b[91m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[91m╸\u001b[0m\u001b[90m━━━\u001b[0m \u001b[32m1.8/2.0 MB\u001b[0m \u001b[31m26.4 MB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m\r\u001b[2K     \u001b[91m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[91m╸\u001b[0m\u001b[90m━━━\u001b[0m \u001b[32m1.8/2.0 MB\u001b[0m \u001b[31m26.4 MB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m\r\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.0/2.0 MB\u001b[0m \u001b[31m14.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 447
        },
        "id": "UwxP6o8pSPbV",
        "outputId": "cdc36707-bafa-4827-bd17-475386adbb1a"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<Axes: >"
            ]
          },
          "metadata": {},
          "execution_count": 6
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 640x480 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAigAAAGdCAYAAAA44ojeAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAqIklEQVR4nO3df3RU9Z3/8Vd+TCYJMIkBMwFNMF1dIQKCIMmo7XY1JGLq+iOnK25KU5ejpzRYIbuo6SLlhxqW/pDiBqg9FOyplC27q60UIWMssK7hV7pYftioqz1xxUm20jD8kMmQfL5/9Jtbx4AyMGE+E56Pc3IO87nv+dzPvW+CL+/MnUkyxhgBAABYJDneCwAAAPgkAgoAALAOAQUAAFiHgAIAAKxDQAEAANYhoAAAAOsQUAAAgHUIKAAAwDqp8V7Auejp6dGhQ4c0ZMgQJSUlxXs5AADgLBhjdPToUY0YMULJyZ9+jSQhA8qhQ4eUn58f72UAAIBz8N577+nyyy//1JqEDChDhgyR9KcD9Hg8MZs3HA6rsbFRZWVlcrlcMZsX/Yu+JSb6lpjoW2KypW/BYFD5+fnOf8c/TUIGlN6XdTweT8wDSmZmpjweD794CYS+JSb6lpjoW2KyrW9n8/YM3iQLAACsQ0ABAADWIaAAAADrEFAAAIB1CCgAAMA6BBQAAGAdAgoAALAOAQUAAFiHgAIAAKwTdUB5//339ZWvfEVDhw5VRkaGxo4dqz179jjbjTGaP3++hg8froyMDJWWluqtt96KmOPw4cOqqqqSx+NRdna2ZsyYoWPHjp3/0QAAgAEhqoDyxz/+UTfeeKNcLpdeeuklHTx4UN/73vd0ySWXODVLly7V8uXLtWrVKu3cuVODBg1SeXm5Tp486dRUVVXpwIED8vv92rhxo7Zv364HHnggdkcFAAASWlTfxfPP//zPys/P15o1a5yxwsJC58/GGC1btkzz5s3THXfcIUn6yU9+Iq/XqxdeeEHTpk3TG2+8oc2bN2v37t2aNGmSJOnpp5/Wbbfdpu9+97saMWJELI4LAAAksKgCyi9/+UuVl5fry1/+srZt26bLLrtM3/jGN3T//fdLkt59910FAgGVlpY6z8nKylJxcbGam5s1bdo0NTc3Kzs72wknklRaWqrk5GTt3LlTd911V5/9hkIhhUIh53EwGJT0py8/CofD0R3xp+idK5Zzov/Rt8RE3xITfUtMtvQtmv1HFVDeeecdrVy5UrW1tfrWt76l3bt365vf/KbS0tJUXV2tQCAgSfJ6vRHP83q9zrZAIKDc3NzIRaSmKicnx6n5pPr6ei1cuLDPeGNjozIzM6M5hLPi9/tjPif6H31LTPQtMdG3xBTvvp04ceKsa6MKKD09PZo0aZKefPJJSdKECRO0f/9+rVq1StXV1dGtMgp1dXWqra11HgeDQeXn56usrEwejydm+wmHw/L7/ZoyZYoVX0cdjTELtsR7CVHbv6A8JvMkct8uZvQtMdG3xGRL33pfATkbUQWU4cOHq6ioKGJs9OjR+vd//3dJUl5eniSpvb1dw4cPd2ra29s1fvx4p6ajoyNijlOnTunw4cPO8z/J7XbL7Xb3GXe5XP1yovtr3v4U6k6K9xKiFutznIh9A31LVPQtMcW7b9HsO6q7eG688Ua1trZGjL355psaOXKkpD+9YTYvL09NTU3O9mAwqJ07d8rn80mSfD6fOjs71dLS4tS88sor6unpUXFxcTTLAQAAA1RUV1DmzJmjG264QU8++aT+9m//Vrt27dIzzzyjZ555RpKUlJSk2bNn6/HHH9dVV12lwsJCPfbYYxoxYoTuvPNOSX+64nLrrbfq/vvv16pVqxQOhzVr1ixNmzaNO3gAAICkKAPK9ddfr+eff151dXVatGiRCgsLtWzZMlVVVTk1Dz/8sI4fP64HHnhAnZ2duummm7R582alp6c7Nc8995xmzZqlW265RcnJyaqsrNTy5ctjd1QAACChRRVQJOlLX/qSvvSlL51xe1JSkhYtWqRFixadsSYnJ0fr1q2LdtcAAOAiwXfxAAAA6xBQAACAdQgoAADAOgQUAABgHQIKAACwDgEFAABYh4ACAACsQ0ABAADWIaAAAADrEFAAAIB1CCgAAMA6BBQAAGAdAgoAALAOAQUAAFiHgAIAAKxDQAEAANYhoAAAAOsQUAAAgHUIKAAAwDoEFAAAYB0CCgAAsA4BBQAAWIeAAgAArENAAQAA1iGgAAAA6xBQAACAdQgoAADAOgQUAABgHQIKAACwDgEFAABYh4ACAACsQ0ABAADWIaAAAADrEFAAAIB1CCgAAMA6BBQAAGAdAgoAALAOAQUAAFiHgAIAAKxDQAEAANYhoAAAAOsQUAAAgHUIKAAAwDoEFAAAYB0CCgAAsA4BBQAAWIeAAgAArENAAQAA1okqoCxYsEBJSUkRP6NGjXK2nzx5UjU1NRo6dKgGDx6syspKtbe3R8zR1tamiooKZWZmKjc3V3PnztWpU6diczQAAGBASI32Cddcc41efvnlP0+Q+ucp5syZo1/96lfasGGDsrKyNGvWLN199936r//6L0lSd3e3KioqlJeXp9dee00ffPCBvvrVr8rlcunJJ5+MweEAAICBIOqAkpqaqry8vD7jR44c0erVq7Vu3TrdfPPNkqQ1a9Zo9OjR2rFjh0pKStTY2KiDBw/q5Zdfltfr1fjx47V48WI98sgjWrBggdLS0s7/iAAAQMKLOqC89dZbGjFihNLT0+Xz+VRfX6+CggK1tLQoHA6rtLTUqR01apQKCgrU3NyskpISNTc3a+zYsfJ6vU5NeXm5Zs6cqQMHDmjChAmn3WcoFFIoFHIeB4NBSVI4HFY4HI72EM6od65YznmhuFNMvJcQtVid50Tu28WMviUm+paYbOlbNPuPKqAUFxdr7dq1uvrqq/XBBx9o4cKF+vznP6/9+/crEAgoLS1N2dnZEc/xer0KBAKSpEAgEBFOerf3bjuT+vp6LVy4sM94Y2OjMjMzozmEs+L3+2M+Z39bOjneK4jepk2bYjpfIvYN9C1R0bfEFO++nThx4qxrowooU6dOdf48btw4FRcXa+TIkfr5z3+ujIyMaKaKSl1dnWpra53HwWBQ+fn5Kisrk8fjidl+wuGw/H6/pkyZIpfLFbN5L4QxC7bEewlR27+gPCbzJHLfLmb0LTHRt8RkS996XwE5G1G/xPNx2dnZ+su//Eu9/fbbmjJlirq6utTZ2RlxFaW9vd15z0peXp527doVMUfvXT6ne19LL7fbLbfb3Wfc5XL1y4nur3n7U6g7Kd5LiFqsz3Ei9g30LVHRt8QU775Fs+/z+hyUY8eO6X/+5380fPhwTZw4US6XS01NTc721tZWtbW1yefzSZJ8Pp/27dunjo4Op8bv98vj8aioqOh8lgIAAAaQqK6g/OM//qNuv/12jRw5UocOHdK3v/1tpaSk6N5771VWVpZmzJih2tpa5eTkyOPx6MEHH5TP51NJSYkkqaysTEVFRZo+fbqWLl2qQCCgefPmqaam5rRXSAAAwMUpqoDyv//7v7r33nv14Ycf6tJLL9VNN92kHTt26NJLL5UkPfXUU0pOTlZlZaVCoZDKy8u1YsUK5/kpKSnauHGjZs6cKZ/Pp0GDBqm6ulqLFi2K7VEBAICEFlVAWb9+/aduT09PV0NDgxoaGs5YM3LkyJjfvQEAAAYWvosHAABYh4ACAACsQ0ABAADWIaAAAADrEFAAAIB1CCgAAMA6BBQAAGAdAgoAALAOAQUAAFiHgAIAAKxDQAEAANYhoAAAAOsQUAAAgHUIKAAAwDoEFAAAYB0CCgAAsA4BBQAAWIeAAgAArENAAQAA1iGgAAAA6xBQAACAdQgoAADAOgQUAABgHQIKAACwDgEFAABYh4ACAACsQ0ABAADWIaAAAADrEFAAAIB1CCgAAMA6BBQAAGAdAgoAALAOAQUAAFiHgAIAAKxDQAEAANYhoAAAAOsQUAAAgHUIKAAAwDoEFAAAYB0CCgAAsA4BBQAAWIeAAgAArENAAQAA1iGgAAAA6xBQAACAdQgoAADAOgQUAABgHQIKAACwznkFlCVLligpKUmzZ892xk6ePKmamhoNHTpUgwcPVmVlpdrb2yOe19bWpoqKCmVmZio3N1dz587VqVOnzmcpAABgADnngLJ792798Ic/1Lhx4yLG58yZoxdffFEbNmzQtm3bdOjQId19993O9u7ublVUVKirq0uvvfaann32Wa1du1bz588/96MAAAADyjkFlGPHjqmqqko/+tGPdMkllzjjR44c0erVq/X9739fN998syZOnKg1a9botdde044dOyRJjY2NOnjwoH76059q/Pjxmjp1qhYvXqyGhgZ1dXXF5qgAAEBCSz2XJ9XU1KiiokKlpaV6/PHHnfGWlhaFw2GVlpY6Y6NGjVJBQYGam5tVUlKi5uZmjR07Vl6v16kpLy/XzJkzdeDAAU2YMKHP/kKhkEKhkPM4GAxKksLhsMLh8Lkcwmn1zhXLOS8Ud4qJ9xKiFqvznMh9u5jRt8RE3xKTLX2LZv9RB5T169frN7/5jXbv3t1nWyAQUFpamrKzsyPGvV6vAoGAU/PxcNK7vXfb6dTX12vhwoV9xhsbG5WZmRntIXwmv98f8zn729LJ8V5B9DZt2hTT+RKxb6BviYq+JaZ49+3EiRNnXRtVQHnvvff00EMPye/3Kz09PeqFnau6ujrV1tY6j4PBoPLz81VWViaPxxOz/YTDYfn9fk2ZMkUulytm814IYxZsifcSorZ/QXlM5knkvl3M6Ftiom+JyZa+9b4CcjaiCigtLS3q6OjQdddd54x1d3dr+/bt+pd/+Rdt2bJFXV1d6uzsjLiK0t7erry8PElSXl6edu3aFTFv710+vTWf5Ha75Xa7+4y7XK5+OdH9NW9/CnUnxXsJUYv1OU7EvoG+JSr6lpji3bdo9h3Vm2RvueUW7du3T3v37nV+Jk2apKqqKufPLpdLTU1NznNaW1vV1tYmn88nSfL5fNq3b586OjqcGr/fL4/Ho6KiomiWAwAABqiorqAMGTJEY8aMiRgbNGiQhg4d6ozPmDFDtbW1ysnJkcfj0YMPPiifz6eSkhJJUllZmYqKijR9+nQtXbpUgUBA8+bNU01NzWmvkgAAgIvPOd3F82meeuopJScnq7KyUqFQSOXl5VqxYoWzPSUlRRs3btTMmTPl8/k0aNAgVVdXa9GiRbFeCgAASFDnHVC2bt0a8Tg9PV0NDQ1qaGg443NGjhwZ8zs4AADAwMF38QAAAOsQUAAAgHUIKAAAwDoEFAAAYB0CCgAAsA4BBQAAWIeAAgAArENAAQAA1iGgAAAA6xBQAACAdQgoAADAOgQUAABgHQIKAACwDgEFAABYh4ACAACsQ0ABAADWIaAAAADrEFAAAIB1CCgAAMA6BBQAAGAdAgoAALAOAQUAAFiHgAIAAKxDQAEAANYhoAAAAOsQUAAAgHUIKAAAwDoEFAAAYB0CCgAAsA4BBQAAWIeAAgAArENAAQAA1iGgAAAA6xBQAACAdQgoAADAOgQUAABgHQIKAACwDgEFAABYh4ACAACsQ0ABAADWIaAAAADrEFAAAIB1CCgAAMA6BBQAAGAdAgoAALAOAQUAAFiHgAIAAKxDQAEAANaJKqCsXLlS48aNk8fjkcfjkc/n00svveRsP3nypGpqajR06FANHjxYlZWVam9vj5ijra1NFRUVyszMVG5urubOnatTp07F5mgAAMCAEFVAufzyy7VkyRK1tLRoz549uvnmm3XHHXfowIEDkqQ5c+boxRdf1IYNG7Rt2zYdOnRId999t/P87u5uVVRUqKurS6+99pqeffZZrV27VvPnz4/tUQEAgISWGk3x7bffHvH4iSee0MqVK7Vjxw5dfvnlWr16tdatW6ebb75ZkrRmzRqNHj1aO3bsUElJiRobG3Xw4EG9/PLL8nq9Gj9+vBYvXqxHHnlECxYsUFpaWuyODAAAJKyoAsrHdXd3a8OGDTp+/Lh8Pp9aWloUDodVWlrq1IwaNUoFBQVqbm5WSUmJmpubNXbsWHm9XqemvLxcM2fO1IEDBzRhwoTT7isUCikUCjmPg8GgJCkcDiscDp/rIfTRO1cs57xQ3Ckm3kuIWqzOcyL37WJG3xITfUtMtvQtmv1HHVD27dsnn8+nkydPavDgwXr++edVVFSkvXv3Ki0tTdnZ2RH1Xq9XgUBAkhQIBCLCSe/23m1nUl9fr4ULF/YZb2xsVGZmZrSH8Jn8fn/M5+xvSyfHewXR27RpU0znS8S+gb4lKvqWmOLdtxMnTpx1bdQB5eqrr9bevXt15MgR/du//Zuqq6u1bdu2aKeJSl1dnWpra53HwWBQ+fn5Kisrk8fjidl+wuGw/H6/pkyZIpfLFbN5L4QxC7bEewlR27+gPCbzJHLfLmb0LTHRt8RkS996XwE5G1EHlLS0NF155ZWSpIkTJ2r37t36wQ9+oHvuuUddXV3q7OyMuIrS3t6uvLw8SVJeXp527doVMV/vXT69Nafjdrvldrv7jLtcrn450f01b38KdSfFewlRi/U5TsS+gb4lKvqWmOLdt2j2fd6fg9LT06NQKKSJEyfK5XKpqanJ2dba2qq2tjb5fD5Jks/n0759+9TR0eHU+P1+eTweFRUVne9SAADAABHVFZS6ujpNnTpVBQUFOnr0qNatW6etW7dqy5YtysrK0owZM1RbW6ucnBx5PB49+OCD8vl8KikpkSSVlZWpqKhI06dP19KlSxUIBDRv3jzV1NSc9goJAAC4OEUVUDo6OvTVr35VH3zwgbKysjRu3Dht2bJFU6ZMkSQ99dRTSk5OVmVlpUKhkMrLy7VixQrn+SkpKdq4caNmzpwpn8+nQYMGqbq6WosWLYrtUQEAgIQWVUBZvXr1p25PT09XQ0ODGhoazlgzcuTImN+9AQAABha+iwcAAFiHgAIAAKxDQAEAANYhoAAAAOsQUAAAgHUIKAAAwDoEFAAAYB0CCgAAsA4BBQAAWIeAAgAArENAAQAA1iGgAAAA6xBQAACAdQgoAADAOgQUAABgHQIKAACwDgEFAABYh4ACAACsQ0ABAADWIaAAAADrEFAAAIB1CCgAAMA6BBQAAGAdAgoAALAOAQUAAFiHgAIAAKxDQAEAANYhoAAAAOsQUAAAgHUIKAAAwDoEFAAAYB0CCgAAsA4BBQAAWIeAAgAArENAAQAA1iGgAAAA6xBQAACAdQgoAADAOgQUAABgHQIKAACwDgEFAABYh4ACAACskxrvBdhozIItCnUnxXsZAABctLiCAgAArENAAQAA1iGgAAAA6xBQAACAdaIKKPX19br++us1ZMgQ5ebm6s4771Rra2tEzcmTJ1VTU6OhQ4dq8ODBqqysVHt7e0RNW1ubKioqlJmZqdzcXM2dO1enTp06/6MBAAADQlQBZdu2baqpqdGOHTvk9/sVDodVVlam48ePOzVz5szRiy++qA0bNmjbtm06dOiQ7r77bmd7d3e3Kioq1NXVpddee03PPvus1q5dq/nz58fuqAAAQEKL6jbjzZs3Rzxeu3atcnNz1dLSoi984Qs6cuSIVq9erXXr1unmm2+WJK1Zs0ajR4/Wjh07VFJSosbGRh08eFAvv/yyvF6vxo8fr8WLF+uRRx7RggULlJaWFrujAwAACem83oNy5MgRSVJOTo4kqaWlReFwWKWlpU7NqFGjVFBQoObmZklSc3Ozxo4dK6/X69SUl5crGAzqwIED57McAAAwQJzzB7X19PRo9uzZuvHGGzVmzBhJUiAQUFpamrKzsyNqvV6vAoGAU/PxcNK7vXfb6YRCIYVCIedxMBiUJIXDYYXD4XM9hD5653Inm5jNiTOLVe9654nl3wX0P/qWmOhbYrKlb9Hs/5wDSk1Njfbv369XX331XKc4a/X19Vq4cGGf8cbGRmVmZsZ8f4sn9cR8TvS1adOmmM7n9/tjOh8uDPqWmOhbYop3306cOHHWtecUUGbNmqWNGzdq+/btuvzyy53xvLw8dXV1qbOzM+IqSnt7u/Ly8pyaXbt2RczXe5dPb80n1dXVqba21nkcDAaVn5+vsrIyeTyeczmE0wqHw/L7/XpsT7JCPXzUfX/bv6A8JvP09m3KlClyuVwxmRP9j74lJvqWmGzpW+8rIGcjqoBijNGDDz6o559/Xlu3blVhYWHE9okTJ8rlcqmpqUmVlZWSpNbWVrW1tcnn80mSfD6fnnjiCXV0dCg3N1fSnxKdx+NRUVHRaffrdrvldrv7jLtcrn450aGeJL6L5wKIde/66+8D+hd9S0z0LTHFu2/R7DuqgFJTU6N169bpF7/4hYYMGeK8ZyQrK0sZGRnKysrSjBkzVFtbq5ycHHk8Hj344IPy+XwqKSmRJJWVlamoqEjTp0/X0qVLFQgENG/ePNXU1Jw2hAAAgItPVAFl5cqVkqQvfvGLEeNr1qzR1772NUnSU089peTkZFVWVioUCqm8vFwrVqxwalNSUrRx40bNnDlTPp9PgwYNUnV1tRYtWnR+RwIAAAaMqF/i+Szp6elqaGhQQ0PDGWtGjhwZ8zdIAgCAgYPv4gEAANYhoAAAAOsQUAAAgHUIKAAAwDoEFAAAYB0CCgAAsA4BBQAAWIeAAgAArENAAQAA1iGgAAAA6xBQAACAdQgoAADAOgQUAABgHQIKAACwDgEFAABYh4ACAACsQ0ABAADWIaAAAADrEFAAAIB1CCgAAMA6BBQAAGAdAgoAALAOAQUAAFiHgAIAAKxDQAEAANYhoAAAAOsQUAAAgHUIKAAAwDoEFAAAYB0CCgAAsA4BBQAAWIeAAgAArENAAQAA1iGgAAAA6xBQAACAdQgoAADAOgQUAABgHQIKAACwDgEFAABYJzXeC8DF64pHfxWTedwpRksnS2MWbFGoOykmc57J75dU9Ov8AIA/4QoKAACwDgEFAABYh4ACAACsQ0ABAADWIaAAAADrEFAAAIB1CCgAAMA6BBQAAGCdqAPK9u3bdfvtt2vEiBFKSkrSCy+8ELHdGKP58+dr+PDhysjIUGlpqd56662ImsOHD6uqqkoej0fZ2dmaMWOGjh07dl4HAgAABo6oA8rx48d17bXXqqGh4bTbly5dquXLl2vVqlXauXOnBg0apPLycp08edKpqaqq0oEDB+T3+7Vx40Zt375dDzzwwLkfBQAAGFCi/qj7qVOnaurUqafdZozRsmXLNG/ePN1xxx2SpJ/85Cfyer164YUXNG3aNL3xxhvavHmzdu/erUmTJkmSnn76ad1222367ne/qxEjRpzH4QAAgIEgpt/F8+677yoQCKi0tNQZy8rKUnFxsZqbmzVt2jQ1NzcrOzvbCSeSVFpaquTkZO3cuVN33XVXn3lDoZBCoZDzOBgMSpLC4bDC4XDM1t87lzvZxGxO9L/efl2IvsXy79vFrvdcck4TC31LTLb0LZr9xzSgBAIBSZLX640Y93q9zrZAIKDc3NzIRaSmKicnx6n5pPr6ei1cuLDPeGNjozIzM2Ox9AiLJ/XEfE70vwvRt02bNvX7Pi42fr8/3kvAOaBviSnefTtx4sRZ1ybEtxnX1dWptrbWeRwMBpWfn6+ysjJ5PJ6Y7SccDsvv9+uxPckK9fTvt+IidtzJRosn9VyQvu1fUN6v819Men/fpkyZIpfLFe/l4CzRt8RkS996XwE5GzENKHl5eZKk9vZ2DR8+3Blvb2/X+PHjnZqOjo6I5506dUqHDx92nv9Jbrdbbre7z7jL5eqXEx3qSVKom4CSaC5E3/gHOfb66/cY/Yu+JaZ49y2afcf0c1AKCwuVl5enpqYmZywYDGrnzp3y+XySJJ/Pp87OTrW0tDg1r7zyinp6elRcXBzL5QAAgAQV9RWUY8eO6e2333Yev/vuu9q7d69ycnJUUFCg2bNn6/HHH9dVV12lwsJCPfbYYxoxYoTuvPNOSdLo0aN166236v7779eqVasUDoc1a9YsTZs2jTt4AACApHMIKHv27NFf//VfO4973xtSXV2ttWvX6uGHH9bx48f1wAMPqLOzUzfddJM2b96s9PR05znPPfecZs2apVtuuUXJycmqrKzU8uXLY3A4AABgIIg6oHzxi1+UMWe+nTMpKUmLFi3SokWLzliTk5OjdevWRbtrAABwkeC7eAAAgHUIKAAAwDoEFAAAYB0CCgAAsA4BBQAAWIeAAgAArENAAQAA1iGgAAAA6xBQAACAdQgoAADAOgQUAABgHQIKAACwTtRfFghczK549FfxXkLUfr+kIt5LAICocQUFAABYh4ACAACsQ0ABAADWIaAAAADrEFAAAIB1CCgAAMA6BBQAAGAdAgoAALAOAQUAAFiHgAIAAKxDQAEAANYhoAAAAOsQUAAAgHUIKAAAwDoEFAAAYB0CCgAAsA4BBQAAWIeAAgAArENAAQAA1kmN9wIA4JOuePRX8V5C1H6/pCLeSwAGFK6gAAAA6xBQAACAdQgoAADAOgQUAABgHQIKAACwDnfxAAOcrXfEuFOMlk6WxizYolB3UryXA8AyXEEBAADWIaAAAADrEFAAAIB1CCgAAMA6BBQAAGAdAgoAALAOAQUAAFiHgAIAAKxDQAEAANaJ6yfJNjQ06Dvf+Y4CgYCuvfZaPf3005o8eXI8lwQA58TWT+z9NL9fUhHvJQBnFLeA8q//+q+qra3VqlWrVFxcrGXLlqm8vFytra3Kzc2N17IAAIgpG8LruXy1RLwDbNxe4vn+97+v+++/X/fdd5+Kioq0atUqZWZm6sc//nG8lgQAACwRlysoXV1damlpUV1dnTOWnJys0tJSNTc396kPhUIKhULO4yNHjkiSDh8+rHA4HLN1hcNhnThxQqnhZHX38OVliSK1x+jEiR76lmDoW/x9+OGHUT+n99/JDz/8UC6Xqx9W9emK65su+D7Plw3fynsuv2/n8vfjsxw9elSSZIz5zNq4nLc//OEP6u7ultfrjRj3er363e9+16e+vr5eCxcu7DNeWFjYb2tEYvm7eC8A54S+xdew78V7BbiQov1968+/H0ePHlVWVtan1tgQ7D5TXV2damtrncc9PT06fPiwhg4dqqSk2P2fVzAYVH5+vt577z15PJ6YzYv+Rd8SE31LTPQtMdnSN2OMjh49qhEjRnxmbVwCyrBhw5SSkqL29vaI8fb2duXl5fWpd7vdcrvdEWPZ2dn9tj6Px8MvXgKib4mJviUm+paYbOjbZ1056RWXN8mmpaVp4sSJamr682uJPT09ampqks/ni8eSAACAReL2Ek9tba2qq6s1adIkTZ48WcuWLdPx48d13333xWtJAADAEnELKPfcc4/+7//+T/Pnz1cgEND48eO1efPmPm+cvZDcbre+/e1v93k5CXajb4mJviUm+paYErFvSeZs7vUBAAC4gPguHgAAYB0CCgAAsA4BBQAAWIeAAgAArENA+ZiGhgZdccUVSk9PV3FxsXbt2hXvJV006uvrdf3112vIkCHKzc3VnXfeqdbW1oiakydPqqamRkOHDtXgwYNVWVnZ58P+2traVFFRoczMTOXm5mru3Lk6depURM3WrVt13XXXye1268orr9TatWv7+/AuCkuWLFFSUpJmz57tjNEzO73//vv6yle+oqFDhyojI0Njx47Vnj17nO3GGM2fP1/Dhw9XRkaGSktL9dZbb0XMcfjwYVVVVcnj8Sg7O1szZszQsWPHImp++9vf6vOf/7zS09OVn5+vpUuXXpDjG4i6u7v12GOPqbCwUBkZGfqLv/gLLV68OOI7bQZc3wyMMcasX7/epKWlmR//+MfmwIED5v777zfZ2dmmvb093ku7KJSXl5s1a9aY/fv3m71795rbbrvNFBQUmGPHjjk1X//6101+fr5pamoye/bsMSUlJeaGG25wtp86dcqMGTPGlJaWmv/+7/82mzZtMsOGDTN1dXVOzTvvvGMyMzNNbW2tOXjwoHn66adNSkqK2bx58wU93oFm165d5oorrjDjxo0zDz30kDNOz+xz+PBhM3LkSPO1r33N7Ny507zzzjtmy5Yt5u2333ZqlixZYrKysswLL7xgXn/9dfM3f/M3prCw0Hz00UdOza233mquvfZas2PHDvOf//mf5sorrzT33nuvs/3IkSPG6/Waqqoqs3//fvOzn/3MZGRkmB/+8IcX9HgHiieeeMIMHTrUbNy40bz77rtmw4YNZvDgweYHP/iBUzPQ+kZA+f8mT55sampqnMfd3d1mxIgRpr6+Po6runh1dHQYSWbbtm3GGGM6OzuNy+UyGzZscGreeOMNI8k0NzcbY4zZtGmTSU5ONoFAwKlZuXKl8Xg8JhQKGWOMefjhh80111wTsa977rnHlJeX9/chDVhHjx41V111lfH7/eav/uqvnIBCz+z0yCOPmJtuuumM23t6ekxeXp75zne+44x1dnYat9ttfvaznxljjDl48KCRZHbv3u3UvPTSSyYpKcm8//77xhhjVqxYYS655BKnj737vvrqq2N9SBeFiooK8/d///cRY3fffbepqqoyxgzMvvESj6Suri61tLSotLTUGUtOTlZpaamam5vjuLKL15EjRyRJOTk5kqSWlhaFw+GIHo0aNUoFBQVOj5qbmzV27NiID/srLy9XMBjUgQMHnJqPz9FbQ5/PXU1NjSoqKvqcV3pmp1/+8peaNGmSvvzlLys3N1cTJkzQj370I2f7u+++q0AgEHHOs7KyVFxcHNG37OxsTZo0yakpLS1VcnKydu7c6dR84QtfUFpamlNTXl6u1tZW/fGPf+zvwxxwbrjhBjU1NenNN9+UJL3++ut69dVXNXXqVEkDs28J8W3G/e0Pf/iDuru7+3yKrdfr1e9+97s4reri1dPTo9mzZ+vGG2/UmDFjJEmBQEBpaWl9viTS6/UqEAg4NafrYe+2T6sJBoP66KOPlJGR0R+HNGCtX79ev/nNb7R79+4+2+iZnd555x2tXLlStbW1+ta3vqXdu3frm9/8ptLS0lRdXe2c99Od84/3JDc3N2J7amqqcnJyImoKCwv7zNG77ZJLLumX4xuoHn30UQWDQY0aNUopKSnq7u7WE088oaqqKkkakH0joMA6NTU12r9/v1599dV4LwWf4r333tNDDz0kv9+v9PT0eC8HZ6mnp0eTJk3Sk08+KUmaMGGC9u/fr1WrVqm6ujrOq8OZ/PznP9dzzz2ndevW6ZprrtHevXs1e/ZsjRgxYsD2jZd4JA0bNkwpKSl97i5ob29XXl5enFZ1cZo1a5Y2btyoX//617r88sud8by8PHV1damzszOi/uM9ysvLO20Pe7d9Wo3H4+H/xKPU0tKijo4OXXfddUpNTVVqaqq2bdum5cuXKzU1VV6vl55ZaPjw4SoqKooYGz16tNra2iT9+bx/2r+HeXl56ujoiNh+6tQpHT58OKre4uzNnTtXjz76qKZNm6axY8dq+vTpmjNnjurr6yUNzL4RUCSlpaVp4sSJampqcsZ6enrU1NQkn88Xx5VdPIwxmjVrlp5//nm98sorfS4xTpw4US6XK6JHra2tamtrc3rk8/m0b9++iF9Av98vj8fj/IPs8/ki5uitoc/Ru+WWW7Rv3z7t3bvX+Zk0aZKqqqqcP9Mz+9x44419buF/8803NXLkSElSYWGh8vLyIs55MBjUzp07I/rW2dmplpYWp+aVV15RT0+PiouLnZrt27crHA47NX6/X1dffTUv75yDEydOKDk58j/ZKSkp6unpkTRA+3bB35ZrqfXr1xu3223Wrl1rDh48aB544AGTnZ0dcXcB+s/MmTNNVlaW2bp1q/nggw+cnxMnTjg1X//6101BQYF55ZVXzJ49e4zP5zM+n8/Z3nvLallZmdm7d6/ZvHmzufTSS097y+rcuXPNG2+8YRoaGrhlNYY+fhePMfTMRrt27TKpqanmiSeeMG+99ZZ57rnnTGZmpvnpT3/q1CxZssRkZ2ebX/ziF+a3v/2tueOOO057u+qECRPMzp07zauvvmquuuqqiNtVOzs7jdfrNdOnTzf79+8369evN5mZmdxmfI6qq6vNZZdd5txm/B//8R9m2LBh5uGHH3ZqBlrfCCgf8/TTT5uCggKTlpZmJk+ebHbs2BHvJV00JJ32Z82aNU7NRx99ZL7xjW+YSy65xGRmZpq77rrLfPDBBxHz/P73vzdTp041GRkZZtiwYeYf/uEfTDgcjqj59a9/bcaPH2/S0tLM5z73uYh94Px8MqDQMzu9+OKLZsyYMcbtdptRo0aZZ555JmJ7T0+Peeyxx4zX6zVut9vccsstprW1NaLmww8/NPfee68ZPHiw8Xg85r777jNHjx6NqHn99dfNTTfdZNxut7nsssvMkiVL+v3YBqpgMGgeeughU1BQYNLT083nPvc580//9E8RtwMPtL4lGfOxj6EDAACwAO9BAQAA1iGgAAAA6xBQAACAdQgoAADAOgQUAABgHQIKAACwDgEFAABYh4ACAACsQ0ABAADWIaAAAADrEFAAAIB1CCgAAMA6/w/HVAgQf9OD3gAAAABJRU5ErkJggg==\n"
          },
          "metadata": {}
        }
      ],
      "source": [
        "import tiktoken\n",
        "\n",
        "# Load the cl100k_base tokenizer which is designed to work with the ada-002 model\n",
        "tokenizer = tiktoken.get_encoding(\"cl100k_base\")\n",
        "\n",
        "df = pd.read_csv('processed/scraped.csv', index_col=0)\n",
        "df.columns = ['title', 'text']\n",
        "\n",
        "# Tokenize the text and save the number of tokens to a new column\n",
        "df['n_tokens'] = df.text.apply(lambda x: len(tokenizer.encode(x)))\n",
        "\n",
        "# Visualize the distribution of the number of tokens per row using a histogram\n",
        "df.n_tokens.hist()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "T9N8LfnuSPbV"
      },
      "outputs": [],
      "source": [
        "max_tokens = 1000\n",
        "\n",
        "# Function to split the text into chunks of a maximum number of tokens\n",
        "def split_into_many(text, max_tokens = max_tokens):\n",
        "\n",
        "    # Split the text into sentences\n",
        "    sentences = text.split('. ')\n",
        "\n",
        "    # Get the number of tokens for each sentence\n",
        "    n_tokens = [len(tokenizer.encode(\" \" + sentence)) for sentence in sentences]\n",
        "\n",
        "    chunks = []\n",
        "    tokens_so_far = 0\n",
        "    chunk = []\n",
        "\n",
        "    # Loop through the sentences and tokens joined together in a tuple\n",
        "    for sentence, token in zip(sentences, n_tokens):\n",
        "\n",
        "        # If the number of tokens so far plus the number of tokens in the current sentence is greater\n",
        "        # than the max number of tokens, then add the chunk to the list of chunks and reset\n",
        "        # the chunk and tokens so far\n",
        "        if tokens_so_far + token > max_tokens:\n",
        "            chunks.append(\". \".join(chunk) + \".\")\n",
        "            chunk = []\n",
        "            tokens_so_far = 0\n",
        "\n",
        "        # If the number of tokens in the current sentence is greater than the max number of\n",
        "        # tokens, go to the next sentence\n",
        "        if token > max_tokens:\n",
        "            continue\n",
        "\n",
        "        # Otherwise, add the sentence to the chunk and add the number of tokens to the total\n",
        "        chunk.append(sentence)\n",
        "        tokens_so_far += token + 1\n",
        "\n",
        "    # Add the last chunk to the list of chunks\n",
        "    if chunk:\n",
        "        chunks.append(\". \".join(chunk) + \".\")\n",
        "\n",
        "    return chunks\n",
        "\n",
        "\n",
        "shortened = []\n",
        "\n",
        "# Loop through the dataframe\n",
        "for row in df.iterrows():\n",
        "\n",
        "    # If the text is None, go to the next row\n",
        "    if row[1]['text'] is None:\n",
        "        continue\n",
        "\n",
        "    # If the number of tokens is greater than the max number of tokens, split the text into chunks\n",
        "    if row[1]['n_tokens'] > max_tokens:\n",
        "        shortened += split_into_many(row[1]['text'])\n",
        "\n",
        "    # Otherwise, add the text to the list of shortened texts\n",
        "    else:\n",
        "        shortened.append( row[1]['text'] )"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 447
        },
        "id": "hgtGoXz-SPbW",
        "outputId": "fb551508-76eb-4868-de16-361a795ebb47"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<Axes: >"
            ]
          },
          "metadata": {},
          "execution_count": 8
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 640x480 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAigAAAGdCAYAAAA44ojeAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAxaElEQVR4nO3df3RU9YH//1cmmUwIMIlBM0NqgtnWChEslFQyan+shkRMXa053eInpWnLkW0arJBd1LRA+aGGZrtKsRFqDwV7lLplt9JKKWQMLaxLCJCK5YdFu7WNW5xkKw0DpEyGzP3+4Te3jgnKwGTmHXg+zsk5zL3vec/7vhwzr3NnbibFsixLAAAABnEkewEAAADvRkEBAADGoaAAAADjUFAAAIBxKCgAAMA4FBQAAGAcCgoAADAOBQUAABgnLdkLOB+RSERHjx7V6NGjlZKSkuzlAACAc2BZlk6cOKG8vDw5HO99jmRYFpSjR48qPz8/2csAAADn4Y033tCVV175nmOGZUEZPXq0pLcP0O12x2XOcDis5uZmlZWVyel0xmVODI6sE4OcE4OcE4esE2Mocw4Gg8rPz7dfx9/LsCwo/W/ruN3uuBaUzMxMud1unvhDjKwTg5wTg5wTh6wTIxE5n8vHM/iQLAAAMA4FBQAAGIeCAgAAjENBAQAAxqGgAAAA41BQAACAcSgoAADAOBQUAABgHAoKAAAwDgUFAAAYh4ICAACMQ0EBAADGoaAAAADjUFAAAIBx0pK9AAAALmZXPfjzZC8hJq5US43XJ3sVnEEBAAAGoqAAAADjUFAAAIBxKCgAAMA4FBQAAGAcCgoAADAOBQUAABiHggIAAIxDQQEAAMahoAAAAONQUAAAgHEoKAAAwDgUFAAAYBwKCgAAME5MBaWvr0+LFi1SYWGhRowYoQ9+8INavny5LMuyx1iWpcWLF2vs2LEaMWKESktL9dprr0XNc+zYMVVVVcntdis7O1uzZ8/WyZMn43NEAABg2IupoHzrW9/S6tWr9d3vflevvPKKvvWtb6mxsVGPP/64PaaxsVGrVq3SmjVr1NbWppEjR6q8vFynT5+2x1RVVenQoUPy+/3avHmzdu7cqTlz5sTvqAAAwLCWFsvgXbt26Y477lBFRYUk6aqrrtKPfvQj7dmzR9LbZ09WrlyphQsX6o477pAk/fCHP5TH49GmTZs0c+ZMvfLKK9q6dav27t2r4uJiSdLjjz+u2267Td/+9reVl5cXz+MDAADDUEwF5YYbbtCTTz6pV199VR/+8If18ssv68UXX9Sjjz4qSXr99dcVCARUWlpq3ycrK0vTpk1Ta2urZs6cqdbWVmVnZ9vlRJJKS0vlcDjU1tamz3zmMwMeNxQKKRQK2beDwaAkKRwOKxwOx3bEZ9E/T7zmw9mRdWKQc2KQc+IM16xdqdb7DzKIy/H2eoci51jmjKmgPPjggwoGgxo/frxSU1PV19enhx9+WFVVVZKkQCAgSfJ4PFH383g89r5AIKDc3NzoRaSlKScnxx7zbg0NDVq6dOmA7c3NzcrMzIzlEN6X3++P63w4O7JODHJODHJOnOGWdeP1yV7B+RmKnHt6es55bEwF5cc//rGeeeYZbdiwQddee63279+vefPmKS8vT9XV1TEv9FzV19errq7Ovh0MBpWfn6+ysjK53e64PEY4HJbf79f06dPldDrjMicGR9aJQc6JQc6JM1yznrhkW7KXEBOXw9Ly4siQ5Nz/Dsi5iKmgLFiwQA8++KBmzpwpSZo0aZL++Mc/qqGhQdXV1fJ6vZKkzs5OjR071r5fZ2enJk+eLEnyer3q6uqKmvfMmTM6duyYff93c7lccrlcA7Y7nc64hzcUc2JwZJ0Y5JwY5Jw4wy3rUF9KspdwXobqNfZcxXQVT09PjxyO6LukpqYqEolIkgoLC+X1etXS0mLvDwaDamtrk8/nkyT5fD51d3ervb3dHrN9+3ZFIhFNmzYtluUAAICLVExnUG6//XY9/PDDKigo0LXXXquXXnpJjz76qL785S9LklJSUjRv3jw99NBDuvrqq1VYWKhFixYpLy9Pd955pyRpwoQJuvXWW3XPPfdozZo1CofDmjt3rmbOnMkVPAAAQFKMBeXxxx/XokWL9NWvflVdXV3Ky8vTP/3TP2nx4sX2mPvvv1+nTp3SnDlz1N3drZtuuklbt25VRkaGPeaZZ57R3Llzdcstt8jhcKiyslKrVq2K31EBAIBhLaaCMnr0aK1cuVIrV64865iUlBQtW7ZMy5YtO+uYnJwcbdiwIZaHBgAAlxC+iwcAABiHggIAAIxDQQEAAMahoAAAAONQUAAAgHEoKAAAwDgUFAAAYBwKCgAAMA4FBQAAGIeCAgAAjENBAQAAxqGgAAAA41BQAACAcSgoAADAOBQUAABgHAoKAAAwDgUFAAAYh4ICAACMQ0EBAADGoaAAAADjUFAAAIBxKCgAAMA4FBQAAGAcCgoAADAOBQUAABiHggIAAIxDQQEAAMahoAAAAONQUAAAgHEoKAAAwDgUFAAAYJyYCspVV12llJSUAT+1tbWSpNOnT6u2tlZjxozRqFGjVFlZqc7Ozqg5Ojo6VFFRoczMTOXm5mrBggU6c+ZM/I4IAAAMezEVlL179+rNN9+0f/x+vyTps5/9rCRp/vz5ev7557Vx40bt2LFDR48e1V133WXfv6+vTxUVFert7dWuXbv01FNPaf369Vq8eHEcDwkAAAx3MRWUK664Ql6v1/7ZvHmzPvjBD+qTn/ykjh8/rrVr1+rRRx/VzTffrKlTp2rdunXatWuXdu/eLUlqbm7W4cOH9fTTT2vy5MmaMWOGli9frqamJvX29g7JAQIAgOEn7Xzv2Nvbq6efflp1dXVKSUlRe3u7wuGwSktL7THjx49XQUGBWltbVVJSotbWVk2aNEkej8ceU15erpqaGh06dEhTpkwZ9LFCoZBCoZB9OxgMSpLC4bDC4fD5HkKU/nniNR/OjqwTg5wTg5wTZ7hm7Uq1kr2EmLgcb693KHKOZc7zLiibNm1Sd3e3vvjFL0qSAoGA0tPTlZ2dHTXO4/EoEAjYY95ZTvr39+87m4aGBi1dunTA9ubmZmVmZp7vIQyq/20rDD2yTgxyTgxyTpzhlnXj9clewfkZipx7enrOeex5F5S1a9dqxowZysvLO98pzll9fb3q6urs28FgUPn5+SorK5Pb7Y7LY4TDYfn9fk2fPl1OpzMuc2JwZJ0Y5JwY5Jw4wzXriUu2JXsJMXE5LC0vjgxJzv3vgJyL8yoof/zjH/XCCy/oJz/5ib3N6/Wqt7dX3d3dUWdROjs75fV67TF79uyJmqv/Kp/+MYNxuVxyuVwDtjudzriHNxRzYnBknRjknBjknDjDLetQX0qyl3Behuo19lyd199BWbdunXJzc1VRUWFvmzp1qpxOp1paWuxtR44cUUdHh3w+nyTJ5/PpwIED6urqssf4/X653W4VFRWdz1IAAMBFKOYzKJFIROvWrVN1dbXS0v5296ysLM2ePVt1dXXKycmR2+3WvffeK5/Pp5KSEklSWVmZioqKNGvWLDU2NioQCGjhwoWqra0d9AwJAAC4NMVcUF544QV1dHToy1/+8oB9jz32mBwOhyorKxUKhVReXq4nnnjC3p+amqrNmzerpqZGPp9PI0eOVHV1tZYtW3ZhRwEAAC4qMReUsrIyWdbgl0xlZGSoqalJTU1NZ73/uHHjtGXLllgfFgAAXEL4Lh4AAGAcCgoAADAOBQUAABiHggIAAIxDQQEAAMahoAAAAONQUAAAgHEoKAAAwDgUFAAAYBwKCgAAMA4FBQAAGIeCAgAAjENBAQAAxqGgAAAA41BQAACAcSgoAADAOBQUAABgHAoKAAAwDgUFAAAYh4ICAACMQ0EBAADGoaAAAADjUFAAAIBxKCgAAMA4FBQAAGAcCgoAADAOBQUAABiHggIAAIxDQQEAAMahoAAAAOPEXFD+9Kc/6fOf/7zGjBmjESNGaNKkSdq3b5+937IsLV68WGPHjtWIESNUWlqq1157LWqOY8eOqaqqSm63W9nZ2Zo9e7ZOnjx54UcDAAAuCjEVlL/85S+68cYb5XQ69Ytf/EKHDx/Wv/3bv+myyy6zxzQ2NmrVqlVas2aN2traNHLkSJWXl+v06dP2mKqqKh06dEh+v1+bN2/Wzp07NWfOnPgdFQAAGNbSYhn8rW99S/n5+Vq3bp29rbCw0P63ZVlauXKlFi5cqDvuuEOS9MMf/lAej0ebNm3SzJkz9corr2jr1q3au3eviouLJUmPP/64brvtNn37299WXl5ePI4LAAAMYzGdQfnZz36m4uJiffazn1Vubq6mTJmi73//+/b+119/XYFAQKWlpfa2rKwsTZs2Ta2trZKk1tZWZWdn2+VEkkpLS+VwONTW1nahxwMAAC4CMZ1B+f3vf6/Vq1errq5OX//617V371597WtfU3p6uqqrqxUIBCRJHo8n6n4ej8feFwgElJubG72ItDTl5OTYY94tFAopFArZt4PBoCQpHA4rHA7Hcghn1T9PvObD2ZF1YpBzYpBz4gzXrF2pVrKXEBOX4+31DkXOscwZU0GJRCIqLi7WI488IkmaMmWKDh48qDVr1qi6ujq2VcagoaFBS5cuHbC9ublZmZmZcX0sv98f1/lwdmSdGOScGOScOMMt68brk72C8zMUOff09Jzz2JgKytixY1VUVBS1bcKECfrP//xPSZLX65UkdXZ2auzYsfaYzs5OTZ482R7T1dUVNceZM2d07Ngx+/7vVl9fr7q6Ovt2MBhUfn6+ysrK5Ha7YzmEswqHw/L7/Zo+fbqcTmdc5sTgyDoxyDkxyDlxhmvWE5dsS/YSYuJyWFpeHBmSnPvfATkXMRWUG2+8UUeOHIna9uqrr2rcuHGS3v7ArNfrVUtLi11IgsGg2traVFNTI0ny+Xzq7u5We3u7pk6dKknavn27IpGIpk2bNujjulwuuVyuAdudTmfcwxuKOTE4sk4Mck4Mck6c4ZZ1qC8l2Us4L0P1GnuuYioo8+fP1w033KBHHnlE//iP/6g9e/boySef1JNPPilJSklJ0bx58/TQQw/p6quvVmFhoRYtWqS8vDzdeeedkt4+43Lrrbfqnnvu0Zo1axQOhzV37lzNnDmTK3gAAICkGAvKxz72MT333HOqr6/XsmXLVFhYqJUrV6qqqsoec//99+vUqVOaM2eOuru7ddNNN2nr1q3KyMiwxzzzzDOaO3eubrnlFjkcDlVWVmrVqlXxOyoAADCsxVRQJOnTn/60Pv3pT591f0pKipYtW6Zly5addUxOTo42bNgQ60MDAIBLBN/FAwAAjENBAQAAxqGgAAAA41BQAACAcSgoAADAOBQUAABgHAoKAAAwDgUFAAAYh4ICAACMQ0EBAADGoaAAAADjUFAAAIBxKCgAAMA4FBQAAGAcCgoAADAOBQUAABiHggIAAIxDQQEAAMahoAAAAONQUAAAgHEoKAAAwDgUFAAAYBwKCgAAMA4FBQAAGIeCAgAAjENBAQAAxqGgAAAA41BQAACAcSgoAADAOBQUAABgHAoKAAAwTkwFZcmSJUpJSYn6GT9+vL3/9OnTqq2t1ZgxYzRq1ChVVlaqs7Mzao6Ojg5VVFQoMzNTubm5WrBggc6cOROfowEAABeFtFjvcO211+qFF1742wRpf5ti/vz5+vnPf66NGzcqKytLc+fO1V133aX//u//liT19fWpoqJCXq9Xu3bt0ptvvqkvfOELcjqdeuSRR+JwOAAA4GIQc0FJS0uT1+sdsP348eNau3atNmzYoJtvvlmStG7dOk2YMEG7d+9WSUmJmpubdfjwYb3wwgvyeDyaPHmyli9frgceeEBLlixRenr6hR8RAAAY9mIuKK+99pry8vKUkZEhn8+nhoYGFRQUqL29XeFwWKWlpfbY8ePHq6CgQK2trSopKVFra6smTZokj8djjykvL1dNTY0OHTqkKVOmDPqYoVBIoVDIvh0MBiVJ4XBY4XA41kMYVP888ZoPZ0fWiUHOiUHOiTNcs3alWsleQkxcjrfXOxQ5xzJnTAVl2rRpWr9+va655hq9+eabWrp0qT7+8Y/r4MGDCgQCSk9PV3Z2dtR9PB6PAoGAJCkQCESVk/79/fvOpqGhQUuXLh2wvbm5WZmZmbEcwvvy+/1xnQ9nR9aJQc6JQc6JM9yybrw+2Ss4P0ORc09PzzmPjamgzJgxw/73ddddp2nTpmncuHH68Y9/rBEjRsQyVUzq6+tVV1dn3w4Gg8rPz1dZWZncbndcHiMcDsvv92v69OlyOp1xmRODI+vEIOfEIOfEGa5ZT1yyLdlLiInLYWl5cWRIcu5/B+RcxPwWzztlZ2frwx/+sH73u99p+vTp6u3tVXd3d9RZlM7OTvszK16vV3v27Imao/8qn8E+19LP5XLJ5XIN2O50OuMe3lDMicGRdWKQc2KQc+IMt6xDfSnJXsJ5GarX2HN1QX8H5eTJk/qf//kfjR07VlOnTpXT6VRLS4u9/8iRI+ro6JDP55Mk+Xw+HThwQF1dXfYYv98vt9utoqKiC1kKAAC4iMR0BuVf/uVfdPvtt2vcuHE6evSovvnNbyo1NVV33323srKyNHv2bNXV1SknJ0dut1v33nuvfD6fSkpKJEllZWUqKirSrFmz1NjYqEAgoIULF6q2tnbQMyQAAODSFFNB+d///V/dfffdeuutt3TFFVfopptu0u7du3XFFVdIkh577DE5HA5VVlYqFAqpvLxcTzzxhH3/1NRUbd68WTU1NfL5fBo5cqSqq6u1bNmy+B4VAAAY1mIqKM8+++x77s/IyFBTU5OamprOOmbcuHHasmVLLA8LAAAuMXwXDwAAMA4FBQAAGIeCAgAAjENBAQAAxqGgAAAA41BQAACAcSgoAADAOBQUAABgHAoKAAAwDgUFAAAYh4ICAACMQ0EBAADGoaAAAADjUFAAAIBxKCgAAMA4FBQAAGAcCgoAADAOBQUAABiHggIAAIxDQQEAAMahoAAAAONQUAAAgHHSkr0AAABiMXHJNoX6UpK9DAwxzqAAAADjUFAAAIBxKCgAAMA4FBQAAGAcCgoAADAOBQUAABiHggIAAIxDQQEAAMa5oIKyYsUKpaSkaN68efa206dPq7a2VmPGjNGoUaNUWVmpzs7OqPt1dHSooqJCmZmZys3N1YIFC3TmzJkLWQoAALiInHdB2bt3r773ve/puuuui9o+f/58Pf/889q4caN27Niho0eP6q677rL39/X1qaKiQr29vdq1a5eeeuoprV+/XosXLz7/owAAABeV8yooJ0+eVFVVlb7//e/rsssus7cfP35ca9eu1aOPPqqbb75ZU6dO1bp167Rr1y7t3r1bktTc3KzDhw/r6aef1uTJkzVjxgwtX75cTU1N6u3tjc9RAQCAYe28vountrZWFRUVKi0t1UMPPWRvb29vVzgcVmlpqb1t/PjxKigoUGtrq0pKStTa2qpJkybJ4/HYY8rLy1VTU6NDhw5pypQpAx4vFAopFArZt4PBoCQpHA4rHA6fzyEM0D9PvObD2ZF1YpBzYpBz4vRn7HJYSV7Jxa0/36F4TscyZ8wF5dlnn9Wvf/1r7d27d8C+QCCg9PR0ZWdnR233eDwKBAL2mHeWk/79/fsG09DQoKVLlw7Y3tzcrMzMzFgP4T35/f64zoezI+vEIOfEIOfEWV4cSfYSLglD8Zzu6ek557ExFZQ33nhD9913n/x+vzIyMmJe2Pmqr69XXV2dfTsYDCo/P19lZWVyu91xeYxwOCy/36/p06fL6XTGZU4MjqwTg5wTg5wTpz/rRfscCkX4NuOh4nJYWl4cGZLndP87IOcipoLS3t6urq4uffSjH7W39fX1aefOnfrud7+rbdu2qbe3V93d3VFnUTo7O+X1eiVJXq9Xe/bsiZq3/yqf/jHv5nK55HK5Bmx3Op1xD28o5sTgyDoxyDkxyDlxQpEUhfooKENtqF5jz1VMH5K95ZZbdODAAe3fv9/+KS4uVlVVlf1vp9OplpYW+z5HjhxRR0eHfD6fJMnn8+nAgQPq6uqyx/j9frndbhUVFcWyHAAAcJGK6QzK6NGjNXHixKhtI0eO1JgxY+zts2fPVl1dnXJycuR2u3XvvffK5/OppKREklRWVqaioiLNmjVLjY2NCgQCWrhwoWprawc9SwIAAC4953UVz3t57LHH5HA4VFlZqVAopPLycj3xxBP2/tTUVG3evFk1NTXy+XwaOXKkqqurtWzZsngvBQAADFMXXFB+9atfRd3OyMhQU1OTmpqaznqfcePGacuWLRf60AAA4CLFd/EAAADjUFAAAIBxKCgAAMA4FBQAAGAcCgoAADAOBQUAABiHggIAAIxDQQEAAMahoAAAAONQUAAAgHEoKAAAwDgUFAAAYBwKCgAAMA4FBQAAGIeCAgAAjENBAQAAxqGgAAAA41BQAACAcSgoAADAOBQUAABgHAoKAAAwDgUFAAAYh4ICAACMQ0EBAADGoaAAAADjUFAAAIBxKCgAAMA4FBQAAGAcCgoAADAOBQUAABgnpoKyevVqXXfddXK73XK73fL5fPrFL35h7z99+rRqa2s1ZswYjRo1SpWVlers7Iyao6OjQxUVFcrMzFRubq4WLFigM2fOxOdoAADARSGmgnLllVdqxYoVam9v1759+3TzzTfrjjvu0KFDhyRJ8+fP1/PPP6+NGzdqx44dOnr0qO666y77/n19faqoqFBvb6927dqlp556SuvXr9fixYvje1QAAGBYS4tl8O233x51++GHH9bq1au1e/duXXnllVq7dq02bNigm2++WZK0bt06TZgwQbt371ZJSYmam5t1+PBhvfDCC/J4PJo8ebKWL1+uBx54QEuWLFF6enr8jgwAAAxbMRWUd+rr69PGjRt16tQp+Xw+tbe3KxwOq7S01B4zfvx4FRQUqLW1VSUlJWptbdWkSZPk8XjsMeXl5aqpqdGhQ4c0ZcqUQR8rFAopFArZt4PBoCQpHA4rHA6f7yFE6Z8nXvPh7Mg6Mcg5Mcg5cfozdjmsJK/k4taf71A8p2OZM+aCcuDAAfl8Pp0+fVqjRo3Sc889p6KiIu3fv1/p6enKzs6OGu/xeBQIBCRJgUAgqpz07+/fdzYNDQ1aunTpgO3Nzc3KzMyM9RDek9/vj+t8ODuyTgxyTgxyTpzlxZFkL+GSMBTP6Z6ennMeG3NBueaaa7R//34dP35c//Ef/6Hq6mrt2LEj1mliUl9fr7q6Ovt2MBhUfn6+ysrK5Ha74/IY4XBYfr9f06dPl9PpjMucGBxZJwY5JwY5J05/1ov2ORSKpCR7ORctl8PS8uLIkDyn+98BORcxF5T09HR96EMfkiRNnTpVe/fu1Xe+8x197nOfU29vr7q7u6POonR2dsrr9UqSvF6v9uzZEzVf/1U+/WMG43K55HK5Bmx3Op1xD28o5sTgyDoxyDkxyDlxQpEUhfooKENtqF5jz9UF/x2USCSiUCikqVOnyul0qqWlxd535MgRdXR0yOfzSZJ8Pp8OHDigrq4ue4zf75fb7VZRUdGFLgUAAFwkYjqDUl9frxkzZqigoEAnTpzQhg0b9Ktf/Urbtm1TVlaWZs+erbq6OuXk5Mjtduvee++Vz+dTSUmJJKmsrExFRUWaNWuWGhsbFQgEtHDhQtXW1g56hgQAAFyaYiooXV1d+sIXvqA333xTWVlZuu6667Rt2zZNnz5dkvTYY4/J4XCosrJSoVBI5eXleuKJJ+z7p6amavPmzaqpqZHP59PIkSNVXV2tZcuWxfeoAADAsBZTQVm7du177s/IyFBTU5OamprOOmbcuHHasmVLLA8LAAAuMXwXDwAAMA4FBQAAGIeCAgAAjENBAQAAxqGgAAAA41BQAACAcSgoAADAOBQUAABgHAoKAAAwDgUFAAAYh4ICAACMQ0EBAADGoaAAAADjUFAAAIBxKCgAAMA4FBQAAGAcCgoAADAOBQUAABiHggIAAIxDQQEAAMahoAAAAONQUAAAgHEoKAAAwDgUFAAAYBwKCgAAMA4FBQAAGIeCAgAAjENBAQAAxqGgAAAA41BQAACAcSgoAADAODEVlIaGBn3sYx/T6NGjlZubqzvvvFNHjhyJGnP69GnV1tZqzJgxGjVqlCorK9XZ2Rk1pqOjQxUVFcrMzFRubq4WLFigM2fOXPjRAACAi0JMBWXHjh2qra3V7t275ff7FQ6HVVZWplOnTtlj5s+fr+eff14bN27Ujh07dPToUd111132/r6+PlVUVKi3t1e7du3SU089pfXr12vx4sXxOyoAADCspcUyeOvWrVG3169fr9zcXLW3t+sTn/iEjh8/rrVr12rDhg26+eabJUnr1q3ThAkTtHv3bpWUlKi5uVmHDx/WCy+8II/Ho8mTJ2v58uV64IEHtGTJEqWnp8fv6AAAwLAUU0F5t+PHj0uScnJyJEnt7e0Kh8MqLS21x4wfP14FBQVqbW1VSUmJWltbNWnSJHk8HntMeXm5ampqdOjQIU2ZMmXA44RCIYVCIft2MBiUJIXDYYXD4Qs5BFv/PPGaD2dH1olBzolBzonTn7HLYSV5JRe3/nyH4jkdy5znXVAikYjmzZunG2+8URMnTpQkBQIBpaenKzs7O2qsx+NRIBCwx7yznPTv7983mIaGBi1dunTA9ubmZmVmZp7vIQzK7/fHdT6cHVknBjknBjknzvLiSLKXcEkYiud0T0/POY8974JSW1urgwcP6sUXXzzfKc5ZfX296urq7NvBYFD5+fkqKyuT2+2Oy2OEw2H5/X5Nnz5dTqczLnNicGSdGOScGOScOP1ZL9rnUCiSkuzlXLRcDkvLiyND8pzufwfkXJxXQZk7d642b96snTt36sorr7S3e71e9fb2qru7O+osSmdnp7xerz1mz549UfP1X+XTP+bdXC6XXC7XgO1OpzPu4Q3FnBgcWScGOScGOSdOKJKiUB8FZagN1WvsuYrpKh7LsjR37lw999xz2r59uwoLC6P2T506VU6nUy0tLfa2I0eOqKOjQz6fT5Lk8/l04MABdXV12WP8fr/cbreKiopiWQ4AALhIxXQGpba2Vhs2bNBPf/pTjR492v7MSFZWlkaMGKGsrCzNnj1bdXV1ysnJkdvt1r333iufz6eSkhJJUllZmYqKijRr1iw1NjYqEAho4cKFqq2tHfQsCQAAuPTEVFBWr14tSfrUpz4VtX3dunX64he/KEl67LHH5HA4VFlZqVAopPLycj3xxBP22NTUVG3evFk1NTXy+XwaOXKkqqurtWzZsgs7EgAAcNGIqaBY1vtf2pWRkaGmpiY1NTWddcy4ceO0ZcuWWB4aAABcQvguHgAAYBwKCgAAMA4FBQAAGIeCAgAAjENBAQAAxqGgAAAA41BQAACAcSgoAADAOBQUAABgHAoKAAAwDgUFAAAYh4ICAACMQ0EBAADGoaAAAADjUFAAAIBxKCgAAMA4FBQAAGAcCgoAADBOWrIXAABIjqse/HmylxATV6qlxuuTvQokCmdQAACAcSgoAADAOBQUAABgHD6DgqSZuGSbQn0pyV5GTP6woiLZSwCASwJnUAAAgHEoKAAAwDgUFAAAYBwKCgAAMA4FBQAAGIereAAgDobjVWmAyTiDAgAAjBNzQdm5c6duv/125eXlKSUlRZs2bYrab1mWFi9erLFjx2rEiBEqLS3Va6+9FjXm2LFjqqqqktvtVnZ2tmbPnq2TJ09e0IEAAICLR8wF5dSpU/rIRz6ipqamQfc3NjZq1apVWrNmjdra2jRy5EiVl5fr9OnT9piqqiodOnRIfr9fmzdv1s6dOzVnzpzzPwoAAHBRifkzKDNmzNCMGTMG3WdZllauXKmFCxfqjjvukCT98Ic/lMfj0aZNmzRz5ky98sor2rp1q/bu3avi4mJJ0uOPP67bbrtN3/72t5WXl3cBhwMAAC4Gcf0Myuuvv65AIKDS0lJ7W1ZWlqZNm6bW1lZJUmtrq7Kzs+1yIkmlpaVyOBxqa2uL53IAAMAwFdereAKBgCTJ4/FEbfd4PPa+QCCg3Nzc6EWkpSknJ8ce826hUEihUMi+HQwGJUnhcFjhcDgua++fJ17z4ez6M3Y5rCSvJHbD6fnBczoxhvPzebjpz5ish1Z/vkPxuyOWOYfFZcYNDQ1aunTpgO3Nzc3KzMyM62P5/f64zoezW14cSfYSYrZly5ZkLyFmPKcTYzg+n4crsk6Mofjd0dPTc85j41pQvF6vJKmzs1Njx461t3d2dmry5Mn2mK6urqj7nTlzRseOHbPv/2719fWqq6uzbweDQeXn56usrExutzsuaw+Hw/L7/Zo+fbqcTmdc5sTg+rNetM+hUGR4/d2Ig0vKk72Ec8ZzOjGG8/N5uHE5LC0vjpD1EOvPeSh+d/S/A3Iu4lpQCgsL5fV61dLSYheSYDCotrY21dTUSJJ8Pp+6u7vV3t6uqVOnSpK2b9+uSCSiadOmDTqvy+WSy+UasN3pdMY9vKGYE4MLRVKG3R+2unpRc7KXcM5cqZYar+c5nSjD8fk8XJF1YgzVa+y5irmgnDx5Ur/73e/s26+//rr279+vnJwcFRQUaN68eXrooYd09dVXq7CwUIsWLVJeXp7uvPNOSdKECRN066236p577tGaNWsUDoc1d+5czZw5kyt4AACApPMoKPv27dPf//3f27f733qprq7W+vXrdf/99+vUqVOaM2eOuru7ddNNN2nr1q3KyMiw7/PMM89o7ty5uuWWW+RwOFRZWalVq1bF4XAAAMDFIOaC8qlPfUqWdfZPUKekpGjZsmVatmzZWcfk5ORow4YNsT40AAC4RPBdPAAAwDjD4jJjvL+rHvx5spdwzvo/vAkAwNlwBgUAABiHggIAAIxDQQEAAMahoAAAAONQUAAAgHEoKAAAwDgUFAAAYBz+DgpwkZu4ZNuw+2K1P6yoSPYSACQZZ1AAAIBxKCgAAMA4FBQAAGAcCgoAADAOBQUAABiHq3gAGIdv5wbAGRQAAGAcCgoAADAOBQUAABiHggIAAIxDQQEAAMahoAAAAONwmfEghtMljgAAXIw4gwIAAIxDQQEAAMahoAAAAONQUAAAgHEoKAAAwDgUFAAAYBwKCgAAMA4FBQAAGCepBaWpqUlXXXWVMjIyNG3aNO3ZsyeZywEAAIZIWkH593//d9XV1emb3/ymfv3rX+sjH/mIysvL1dXVlawlAQAAQyStoDz66KO655579KUvfUlFRUVas2aNMjMz9YMf/CBZSwIAAIZIynfx9Pb2qr29XfX19fY2h8Oh0tJStba2DhgfCoUUCoXs28ePH5ckHTt2TOFwOC5rCofD6unp0VtvvaW0M6fiMicGlxax1NMTUVrYob5ISrKXc9Ei58Qg58Qh68Toz/mtt96S0+mM69wnTpyQJFmW9f7riOsjn6M///nP6uvrk8fjidru8Xj029/+dsD4hoYGLV26dMD2wsLCIVsjhtb/S/YCLhHknBjknDhknRhDnfOJEyeUlZX1nmOGxbcZ19fXq66uzr4diUR07NgxjRkzRikp8WnRwWBQ+fn5euONN+R2u+MyJwZH1olBzolBzolD1okxlDlblqUTJ04oLy/vfccmpaBcfvnlSk1NVWdnZ9T2zs5Oeb3eAeNdLpdcLlfUtuzs7CFZm9vt5omfIGSdGOScGOScOGSdGEOV8/udOemXlA/Jpqena+rUqWppabG3RSIRtbS0yOfzJWNJAADAIEl7i6eurk7V1dUqLi7W9ddfr5UrV+rUqVP60pe+lKwlAQAAQyStoHzuc5/T//3f/2nx4sUKBAKaPHmytm7dOuCDs4nicrn0zW9+c8BbSYg/sk4Mck4Mck4csk4MU3JOsc7lWh8AAIAE4rt4AACAcSgoAADAOBQUAABgHAoKAAAwDgXl/9fU1KSrrrpKGRkZmjZtmvbs2ZPsJQ0rDQ0N+tjHPqbRo0crNzdXd955p44cORI15vTp06qtrdWYMWM0atQoVVZWDvhjfR0dHaqoqFBmZqZyc3O1YMECnTlzJpGHMqysWLFCKSkpmjdvnr2NnOPjT3/6kz7/+c9rzJgxGjFihCZNmqR9+/bZ+y3L0uLFizV27FiNGDFCpaWleu2116LmOHbsmKqqquR2u5Wdna3Zs2fr5MmTiT4Uo/X19WnRokUqLCzUiBEj9MEPflDLly+P+q4Wso7dzp07dfvttysvL08pKSnatGlT1P54Zfqb3/xGH//4x5WRkaH8/Hw1NjbG7yAsWM8++6yVnp5u/eAHP7AOHTpk3XPPPVZ2drbV2dmZ7KUNG+Xl5da6deusgwcPWvv377duu+02q6CgwDp58qQ95itf+YqVn59vtbS0WPv27bNKSkqsG264wd5/5swZa+LEiVZpaan10ksvWVu2bLEuv/xyq76+PhmHZLw9e/ZYV111lXXddddZ9913n72dnC/csWPHrHHjxllf/OIXrba2Nuv3v/+9tW3bNut3v/udPWbFihVWVlaWtWnTJuvll1+2/uEf/sEqLCy0/vrXv9pjbr31VusjH/mItXv3buu//uu/rA996EPW3XffnYxDMtbDDz9sjRkzxtq8ebP1+uuvWxs3brRGjRplfec737HHkHXstmzZYn3jG9+wfvKTn1iSrOeeey5qfzwyPX78uOXxeKyqqirr4MGD1o9+9CNrxIgR1ve+9724HAMFxbKs66+/3qqtrbVv9/X1WXl5eVZDQ0MSVzW8dXV1WZKsHTt2WJZlWd3d3ZbT6bQ2btxoj3nllVcsSVZra6tlWW//D+VwOKxAIGCPWb16teV2u61QKJTYAzDciRMnrKuvvtry+/3WJz/5SbugkHN8PPDAA9ZNN9101v2RSMTyer3Wv/7rv9rburu7LZfLZf3oRz+yLMuyDh8+bEmy9u7da4/5xS9+YaWkpFh/+tOfhm7xw0xFRYX15S9/OWrbXXfdZVVVVVmWRdbx8O6CEq9Mn3jiCeuyyy6L+r3xwAMPWNdcc01c1n3Jv8XT29ur9vZ2lZaW2tscDodKS0vV2tqaxJUNb8ePH5ck5eTkSJLa29sVDoejch4/frwKCgrsnFtbWzVp0qSoP9ZXXl6uYDCoQ4cOJXD15qutrVVFRUVUnhI5x8vPfvYzFRcX67Of/axyc3M1ZcoUff/737f3v/766woEAlE5Z2Vladq0aVE5Z2dnq7i42B5TWloqh8Ohtra2xB2M4W644Qa1tLTo1VdflSS9/PLLevHFFzVjxgxJZD0U4pVpa2urPvGJTyg9Pd0eU15eriNHjugvf/nLBa9zWHyb8VD685//rL6+vgF/wdbj8ei3v/1tklY1vEUiEc2bN0833nijJk6cKEkKBAJKT08f8CWPHo9HgUDAHjPYf4f+fXjbs88+q1//+tfau3fvgH3kHB+///3vtXr1atXV1enrX/+69u7dq6997WtKT09XdXW1ndNgOb4z59zc3Kj9aWlpysnJIed3ePDBBxUMBjV+/Hilpqaqr69PDz/8sKqqqiSJrIdAvDINBAIqLCwcMEf/vssuu+yC1nnJFxTEX21trQ4ePKgXX3wx2Uu56Lzxxhu677775Pf7lZGRkezlXLQikYiKi4v1yCOPSJKmTJmigwcPas2aNaqurk7y6i4uP/7xj/XMM89ow4YNuvbaa7V//37NmzdPeXl5ZH2Ju+Tf4rn88suVmpo64CqHzs5Oeb3eJK1q+Jo7d642b96sX/7yl7ryyivt7V6vV729veru7o4a/86cvV7voP8d+vfh7bdwurq69NGPflRpaWlKS0vTjh07tGrVKqWlpcnj8ZBzHIwdO1ZFRUVR2yZMmKCOjg5Jf8vpvX5veL1edXV1Re0/c+aMjh07Rs7vsGDBAj344IOaOXOmJk2apFmzZmn+/PlqaGiQRNZDIV6ZDvXvkku+oKSnp2vq1KlqaWmxt0UiEbW0tMjn8yVxZcOLZVmaO3eunnvuOW3fvn3Aab+pU6fK6XRG5XzkyBF1dHTYOft8Ph04cCDqfwq/3y+32z3gxeJSdcstt+jAgQPav3+//VNcXKyqqir73+R84W688cYBl8m/+uqrGjdunCSpsLBQXq83KudgMKi2traonLu7u9Xe3m6P2b59uyKRiKZNm5aAoxgeenp65HBEvxSlpqYqEolIIuuhEK9MfT6fdu7cqXA4bI/x+/265pprLvjtHUlcZmxZb19m7HK5rPXr11uHDx+25syZY2VnZ0dd5YD3VlNTY2VlZVm/+tWvrDfffNP+6enpscd85StfsQoKCqzt27db+/bts3w+n+Xz+ez9/Ze/lpWVWfv377e2bt1qXXHFFVz++j7eeRWPZZFzPOzZs8dKS0uzHn74Yeu1116znnnmGSszM9N6+umn7TErVqywsrOzrZ/+9KfWb37zG+uOO+4Y9DLNKVOmWG1tbdaLL75oXX311Zf0pa+Dqa6utj7wgQ/Ylxn/5Cc/sS6//HLr/vvvt8eQdexOnDhhvfTSS9ZLL71kSbIeffRR66WXXrL++Mc/WpYVn0y7u7stj8djzZo1yzp48KD17LPPWpmZmVxmHG+PP/64VVBQYKWnp1vXX3+9tXv37mQvaViRNOjPunXr7DF//etfra9+9avWZZddZmVmZlqf+cxnrDfffDNqnj/84Q/WjBkzrBEjRliXX3659c///M9WOBxO8NEML+8uKOQcH88//7w1ceJEy+VyWePHj7eefPLJqP2RSMRatGiR5fF4LJfLZd1yyy3WkSNHosa89dZb1t13322NGjXKcrvd1pe+9CXrxIkTiTwM4wWDQeu+++6zCgoKrIyMDOvv/u7vrG984xtRl66Sdex++ctfDvo7ubq62rKs+GX68ssvWzfddJPlcrmsD3zgA9aKFSvidgwplvWOP9cHAABggEv+MygAAMA8FBQAAGAcCgoAADAOBQUAABiHggIAAIxDQQEAAMahoAAAAONQUAAAgHEoKAAAwDgUFAAAYBwKCgAAMA4FBQAAGOf/Azo6EP7KGXHaAAAAAElFTkSuQmCC\n"
          },
          "metadata": {}
        }
      ],
      "source": [
        "df = pd.DataFrame(shortened, columns = ['text'])\n",
        "df['n_tokens'] = df.text.apply(lambda x: len(tokenizer.encode(x)))\n",
        "df.n_tokens.hist()"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install openai --quiet"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "YAF-UCzXWpFt",
        "outputId": "375122a4-0e75-4d38-f3b8-540b3e58b48f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[?25l     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m0.0/76.5 kB\u001b[0m \u001b[31m?\u001b[0m eta \u001b[36m-:--:--\u001b[0m\r\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m76.5/76.5 kB\u001b[0m \u001b[31m2.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "44JlQDaYSPbW",
        "outputId": "98c61927-0858-4285-974e-3a397795e57a"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                                text  n_tokens  \\\n",
              "0  blog openai lp.  OpenAI LP    CloseSearch Subm...       963   \n",
              "1  Our structure gives us flexibility for how to ...       639   \n",
              "2  blog dall e introducing outpainting.  DALL·E: ...       734   \n",
              "3  blog introducing the chatgpt app for ios.  Int...       547   \n",
              "4  careers software engineer infrastructure.  Sof...       907   \n",
              "\n",
              "                                          embeddings  \n",
              "0  [0.00437886081635952, -0.023002535104751587, -...  \n",
              "1  [0.004697161260992289, -0.02560965158045292, 0...  \n",
              "2  [-0.013597782701253891, 0.0023327525705099106,...  \n",
              "3  [-0.009141863323748112, -0.002781352959573269,...  \n",
              "4  [0.017988545820116997, -0.018912095576524734, ...  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-e0fe58ff-ace0-4071-877c-34b171815f7c\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "      <th>n_tokens</th>\n",
              "      <th>embeddings</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>blog openai lp.  OpenAI LP    CloseSearch Subm...</td>\n",
              "      <td>963</td>\n",
              "      <td>[0.00437886081635952, -0.023002535104751587, -...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Our structure gives us flexibility for how to ...</td>\n",
              "      <td>639</td>\n",
              "      <td>[0.004697161260992289, -0.02560965158045292, 0...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>blog dall e introducing outpainting.  DALL·E: ...</td>\n",
              "      <td>734</td>\n",
              "      <td>[-0.013597782701253891, 0.0023327525705099106,...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>blog introducing the chatgpt app for ios.  Int...</td>\n",
              "      <td>547</td>\n",
              "      <td>[-0.009141863323748112, -0.002781352959573269,...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>careers software engineer infrastructure.  Sof...</td>\n",
              "      <td>907</td>\n",
              "      <td>[0.017988545820116997, -0.018912095576524734, ...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-e0fe58ff-ace0-4071-877c-34b171815f7c')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-e0fe58ff-ace0-4071-877c-34b171815f7c button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-e0fe58ff-ace0-4071-877c-34b171815f7c');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-1be0fb21-3841-4d59-b2e7-51baf9fbbda3\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-1be0fb21-3841-4d59-b2e7-51baf9fbbda3')\"\n",
              "            title=\"Suggest charts.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-1be0fb21-3841-4d59-b2e7-51baf9fbbda3 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "    </div>\n",
              "  </div>\n"
            ]
          },
          "metadata": {},
          "execution_count": 10
        }
      ],
      "source": [
        "import openai\n",
        "openai.api_key = \"sk-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\"\n",
        "df['embeddings'] = df.text.apply(lambda x: openai.Embedding.create(input=x, engine='text-embedding-ada-002')['data'][0]['embedding'])\n",
        "df.to_csv('processed/embeddings.csv')\n",
        "df.head()"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df.shape"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "96yuNhesWysZ",
        "outputId": "f5f59c10-acb3-422c-cf72-b80dab8eb977"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(1980, 3)"
            ]
          },
          "metadata": {},
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "bv5-NXXGSPbW"
      },
      "outputs": [],
      "source": [
        "from openai.embeddings_utils import distances_from_embeddings\n",
        "\n",
        "df['embeddings'] = df.text.apply(lambda x: openai.Embedding.create(input=x, engine='text-embedding-ada-002')['data'][0]['embedding'])\n",
        "\n",
        "df.to_csv('processed/embeddings.csv')\n",
        "df.head()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "JJfSJtmpSPbX",
        "outputId": "72931d73-5098-4e5c-c339-9e9c06e3fdc3"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "      <th>n_tokens</th>\n",
              "      <th>embeddings</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>blog authors maddie.   Maddie Hall - OpenAI   ...</td>\n",
              "      <td>175</td>\n",
              "      <td>[-0.012958061881363392, -0.006103983614593744,...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>blog authors tom.   Tom Brown - OpenAI        ...</td>\n",
              "      <td>228</td>\n",
              "      <td>[-0.0053874170407652855, -0.009962032549083233...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>blog openai scholars 2019 final projects.   Op...</td>\n",
              "      <td>492</td>\n",
              "      <td>[0.0019150723237544298, -0.0070442273281514645...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>In this project, I used curiosity-driven explo...</td>\n",
              "      <td>478</td>\n",
              "      <td>[-0.0067560747265815735, 0.0004431474662851542...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Results revealed that the optimal RL policies ...</td>\n",
              "      <td>499</td>\n",
              "      <td>[-0.012868616729974747, 0.0029640409629791975,...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                text  n_tokens  \\\n",
              "0  blog authors maddie.   Maddie Hall - OpenAI   ...       175   \n",
              "1  blog authors tom.   Tom Brown - OpenAI        ...       228   \n",
              "2  blog openai scholars 2019 final projects.   Op...       492   \n",
              "3  In this project, I used curiosity-driven explo...       478   \n",
              "4  Results revealed that the optimal RL policies ...       499   \n",
              "\n",
              "                                          embeddings  \n",
              "0  [-0.012958061881363392, -0.006103983614593744,...  \n",
              "1  [-0.0053874170407652855, -0.009962032549083233...  \n",
              "2  [0.0019150723237544298, -0.0070442273281514645...  \n",
              "3  [-0.0067560747265815735, 0.0004431474662851542...  \n",
              "4  [-0.012868616729974747, 0.0029640409629791975,...  "
            ]
          },
          "execution_count": 31,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "from ast import literal_eval\n",
        "from openai.embeddings_utils import distances_from_embeddings, cosine_similarity\n",
        "\n",
        "df=pd.read_csv('processed/embeddings.csv', index_col=0)\n",
        "df['embeddings'] = df['embeddings'].apply(literal_eval).apply(np.array)\n",
        "\n",
        "df.head()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Hj55ICxbSPbX",
        "outputId": "311b9536-94b6-4864-b343-b1d9a68d8d8c"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "'No, you are not allowed to publish model outputs to Twitter without a human review. You must manually review each generation before sharing or while streaming, and indicate that the content is AI-generated in a way no user could reasonably miss or misunderstand.'"
            ]
          },
          "execution_count": 32,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "def create_context(\n",
        "    question, df, max_len=1800, size=\"ada\"\n",
        "):\n",
        "    \"\"\"\n",
        "    Create a context for a question by finding the most similar context from the dataframe\n",
        "    \"\"\"\n",
        "\n",
        "    # Get the embeddings for the question\n",
        "    q_embeddings = openai.Embedding.create(input=question, engine='text-embedding-ada-002')['data'][0]['embedding']\n",
        "\n",
        "    # Get the distances from the embeddings\n",
        "    df['distances'] = distances_from_embeddings(q_embeddings, df['embeddings'].values, distance_metric='cosine')\n",
        "\n",
        "\n",
        "    returns = []\n",
        "    cur_len = 0\n",
        "\n",
        "    # Sort by distance and add the text to the context until the context is too long\n",
        "    for i, row in df.sort_values('distances', ascending=True).iterrows():\n",
        "\n",
        "        # Add the length of the text to the current length\n",
        "        cur_len += row['n_tokens'] + 4\n",
        "\n",
        "        # If the context is too long, break\n",
        "        if cur_len > max_len:\n",
        "            break\n",
        "\n",
        "        # Else add it to the text that is being returned\n",
        "        returns.append(row[\"text\"])\n",
        "\n",
        "    # Return the context\n",
        "    return \"\\n\\n###\\n\\n\".join(returns)\n",
        "\n",
        "def answer_question(\n",
        "    df,\n",
        "    model=\"text-davinci-003\",\n",
        "    question=\"Am I allowed to publish model outputs to Twitter, without a human review?\",\n",
        "    max_len=1800,\n",
        "    size=\"ada\",\n",
        "    debug=False,\n",
        "    max_tokens=150,\n",
        "    stop_sequence=None\n",
        "):\n",
        "    \"\"\"\n",
        "    Answer a question based on the most similar context from the dataframe texts\n",
        "    \"\"\"\n",
        "    context = create_context(\n",
        "        question,\n",
        "        df,\n",
        "        max_len=max_len,\n",
        "        size=size,\n",
        "    )\n",
        "    # If debug, print the raw model response\n",
        "    if debug:\n",
        "        print(\"Context:\\n\" + context)\n",
        "        print(\"\\n\\n\")\n",
        "\n",
        "    try:\n",
        "        # Create a completions using the question and context\n",
        "        response = openai.Completion.create(\n",
        "            prompt=f\"Answer the question based on the context below, and if the question can't be answered based on the context, say \\\"I don't know\\\"\\n\\nContext: {context}\\n\\n---\\n\\nQuestion: {question}\\nAnswer:\",\n",
        "            temperature=0,\n",
        "            max_tokens=max_tokens,\n",
        "            top_p=1,\n",
        "            frequency_penalty=0,\n",
        "            presence_penalty=0,\n",
        "            stop=stop_sequence,\n",
        "            model=model,\n",
        "        )\n",
        "        return response[\"choices\"][0][\"text\"].strip()\n",
        "    except Exception as e:\n",
        "        print(e)\n",
        "        return \"\""
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "neEZOJJYSPbX",
        "outputId": "bc53e44e-8e07-400d-ef7b-8c090d1038ed"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "\"I don't know.\""
            ]
          },
          "execution_count": 33,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "answer_question(df, question=\"What day is it?\", debug=False)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Bx3j90Q2SPbX",
        "outputId": "a0269692-8ac0-47d4-ac1c-f2478919c137"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "'The newest embeddings model is text-embedding-ada-002.'"
            ]
          },
          "execution_count": 34,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "answer_question(df, question=\"What is our newest embeddings model?\")"
      ]
    }
  ],
  "metadata": {
    "kernelspec": {
      "display_name": "env",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.9.6"
    },
    "orig_nbformat": 4,
    "vscode": {
      "interpreter": {
        "hash": "05f34a34d73b71652304030c1097be3a5720ea2447153dd6542d145a26b73181"
      }
    },
    "colab": {
      "provenance": [],
      "include_colab_link": true
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}