gemma.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "authorship_tag": "ABX9TyM9SOW/NdIEbbu6w7IbxxFG",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/zhaizeyu/test/blob/main/gemma.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install --upgrade accelerate"
      ],
      "metadata": {
        "id": "wpZtPPBSl5wg"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "5ayKYvUrhRP1"
      },
      "outputs": [],
      "source": [
        "import os\n",
        "from google.colab import userdata\n",
        "\n",
        "# Note: `userdata.get` is a Colab API. If you're not using Colab, set the env\n",
        "# vars as appropriate for your system.\n",
        "os.environ[\"KAGGLE_USERNAME\"] = userdata.get('KAGGLE_USERNAME')\n",
        "os.environ[\"KAGGLE_KEY\"] = userdata.get('KAGGLE_KEY')\n",
        "\n",
        "os.environ[\"KERAS_BACKEND\"] = \"torch\"\n",
        "os.environ[\"XLA_PYTHON_CLIENT_MEM_FRACTION\"]=\"1.00\""
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install -U keras-nlp\n",
        "!pip install -U keras>=3"
      ],
      "metadata": {
        "id": "Q0rH0Dwghk98"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import keras\n",
        "import keras_nlp\n",
        "import numpy as np\n",
        "import accelerate"
      ],
      "metadata": {
        "id": "Yo2V8LnMhfa4"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!set PYTORCH_CUDA_ALLOC_CONF=max_split_size_mb:32"
      ],
      "metadata": {
        "id": "dO7ruEFgDnNe"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "gemma_lm = keras_nlp.models.GemmaCausalLM.from_preset(\"gemma_2b_en\")"
      ],
      "metadata": {
        "id": "DmHFnuvfhuwo"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "gemma_lm.summary()"
      ],
      "metadata": {
        "id": "xFJ9SWD1m2nH"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "gemma_lm.generate(\"Keras is a\", max_length=30)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        },
        "id": "AwIlwiCTmUV1",
        "outputId": "fadbd666-921d-4ee6-cc49-2f40a0bae7b5"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'Keras is a popular deep learning library for Python. It is easy to use and has a wide range of features. One of the features of'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!wget -o databricks-dolly-15k.jsonl https://huggingface.co/datasets/databricks/databricks-dolly-15k/resolve/main/databricks-dolly-15k.jsonl"
      ],
      "metadata": {
        "id": "E-QMaqQ2mmbq"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import json\n",
        "data=[]\n",
        "with open(\"databricks-dolly-15k.jsonl.1\") as file:\n",
        "    for line in file:\n",
        "        features = json.loads(line)\n",
        "        # Filter out examples with context, to keep it simple.\n",
        "        if features[\"context\"]:\n",
        "            continue\n",
        "        # Format the entire example as a single string.\n",
        "        template = \"Instruction:\\n{instruction}\\n\\nResponse:\\n{response}\"\n",
        "        data.append(template.format(**features))\n",
        "\n",
        "# Only use 1000 training examples, to keep it fast.\n",
        "data = data[:100]"
      ],
      "metadata": {
        "id": "jjEFAqDEmzic"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "prompt = template.format(\n",
        "    instruction=\"What should I do on a trip to Europe?\",\n",
        "    response=\"\",\n",
        ")\n",
        "sampler = keras_nlp.samplers.TopKSampler(k=5, seed=2)\n",
        "gemma_lm.compile(sampler=sampler)\n",
        "print(gemma_lm.generate(prompt, max_length=256))"
      ],
      "metadata": {
        "id": "bPMueA_enkCM"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Enable LoRA for the model and set the LoRA rank to 4.\n",
        "gemma_lm.backbone.enable_lora(rank=4)\n",
        "gemma_lm.summary()"
      ],
      "metadata": {
        "id": "ZBp0O7XSomEl"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Limit the input sequence length to 512 (to control memory usage).\n",
        "gemma_lm.preprocessor.sequence_length = 512\n",
        "# Use AdamW (a common optimizer for transformer models).\n",
        "optimizer = keras.optimizers.AdamW(\n",
        "    learning_rate=5e-5,\n",
        "    weight_decay=0.01,\n",
        ")\n",
        "# Exclude layernorm and bias terms from decay.\n",
        "optimizer.exclude_from_weight_decay(var_names=[\"bias\", \"scale\"])\n",
        "\n",
        "gemma_lm.compile(\n",
        "    loss=keras.losses.SparseCategoricalCrossentropy(from_logits=True),\n",
        "    optimizer=optimizer,\n",
        "    weighted_metrics=[keras.metrics.SparseCategoricalAccuracy()],\n",
        ")\n",
        "gemma_lm.fit(data, epochs=1, batch_size=1)"
      ],
      "metadata": {
        "id": "ZZA-OJtGou0Z"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!free -h\n",
        "\n"
      ],
      "metadata": {
        "id": "aced1hmxEaW1"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}