split-lang-demo.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {},
      "outputs": [],
      "source": [
        "%%capture\n",
        "%pip install split-lang==2.0.5"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {},
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "c:\\Users\\admin\\.conda\\envs\\melotts\\lib\\site-packages\\tqdm\\auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
            "  from .autonotebook import tqdm as notebook_tqdm\n"
          ]
        }
      ],
      "source": [
        "from split_lang import LangSplitter"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {},
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Building prefix dict from c:\\Users\\admin\\.conda\\envs\\melotts\\lib\\site-packages\\wordfreq\\data\\jieba_zh.txt ...\n",
            "2024-10-31 23:07:24,535  DEBUG [jieba]: Building prefix dict from c:\\Users\\admin\\.conda\\envs\\melotts\\lib\\site-packages\\wordfreq\\data\\jieba_zh.txt ...\n",
            "Loading model from cache C:\\Users\\admin\\AppData\\Local\\Temp\\jieba.u4a100f98a875a2927980f42a20318bd7.cache\n",
            "2024-10-31 23:07:24,537  DEBUG [jieba]: Loading model from cache C:\\Users\\admin\\AppData\\Local\\Temp\\jieba.u4a100f98a875a2927980f42a20318bd7.cache\n",
            "Loading model cost 0.059 seconds.\n",
            "2024-10-31 23:07:24,597  DEBUG [jieba]: Loading model cost 0.059 seconds.\n",
            "Prefix dict has been built successfully.\n",
            "2024-10-31 23:07:24,597  DEBUG [jieba]: Prefix dict has been built successfully.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "0|zh:你喜欢看\n",
            "1|ja:アニメ\n",
            "2|zh:吗\n"
          ]
        }
      ],
      "source": [
        "text = \"你喜欢看アニメ吗\"\n",
        "lang_splitter = LangSplitter()\n",
        "substr = lang_splitter.split_by_lang(\n",
        "    text=text,\n",
        ")\n",
        "for index, item in enumerate(substr):\n",
        "    print(f\"{index}|{item.lang}:{item.text}\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "metadata": {},
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "0|zh:你喜欢看\n",
            "1|ja:アニメ\n",
            "2|zh:吗\n",
            "3|punctuation:？\n",
            "4|zh:我也喜欢看\n",
            "----------------------\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.OTHERS\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.OTHERS\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "0|en:Please star this project on GitHub\n",
            "1|punctuation:, \n",
            "2|en:Thanks you\n",
            "3|punctuation:. \n",
            "4|en:I love you\n",
            "5|zh:请加星这个项目\n",
            "6|punctuation:，\n",
            "7|zh:谢谢你\n",
            "8|punctuation:。\n",
            "9|zh:我爱你\n",
            "10|ja:この項目をスターしてください\n",
            "11|punctuation:、\n",
            "12|ja:ありがとうございます\n",
            "13|punctuation:！\n",
            "14|ja:愛してる\n",
            "----------------------\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "0|ja:日本語\n",
            "1|punctuation:（\n",
            "2|ja:にほんご\n",
            "3|punctuation:、\n",
            "4|ja:にっぽんご\n",
            "5|punctuation:）\n",
            "6|ja:は\n",
            "7|punctuation:、\n",
            "8|ja:日本国内や\n",
            "9|punctuation:、\n",
            "10|ja:かつての日本領だった国\n",
            "11|punctuation:、\n",
            "12|ja:そして国外移民や移住者を含む日本人同士の間で使用されている言語\n",
            "13|punctuation:。\n",
            "14|ja:日本は法令によって公用語を規定していないが\n",
            "15|punctuation:、\n",
            "16|ja:法令その他の公用文は全て日本語で記述され\n",
            "17|punctuation:、\n",
            "18|ja:各種法令において日本語を用いることが規定され\n",
            "19|punctuation:、\n",
            "20|ja:学校教育においては\n",
            "21|punctuation:「\n",
            "22|ja:国語\n",
            "23|punctuation:」\n",
            "24|ja:の教科として学習を行うなど\n",
            "25|punctuation:、\n",
            "26|ja:事実上日本国内において唯一の公用語となっている\n",
            "27|punctuation:。\n",
            "----------------------\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "0|zh:日语是日本通用语及事实上的官方语言\n",
            "1|punctuation:。\n",
            "2|zh:没有精确的日语使用人口的统计\n",
            "3|punctuation:，\n",
            "4|zh:如果计算日本人口以及居住在日本以外的日本人\n",
            "5|punctuation:、\n",
            "6|zh:日侨和日裔\n",
            "7|punctuation:，\n",
            "8|zh:日语使用者应超过一亿三千万人\n",
            "9|punctuation:。\n",
            "----------------------\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.DIGIT\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.DIGIT\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.DIGIT\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.OTHERS\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.DIGIT\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.DIGIT\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.OTHERS\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.OTHERS\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.DIGIT\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.DIGIT\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "0|ja:入学早々隣のクラスの絵奈は男子からの人気者\n",
            "1|punctuation:。\n",
            "2|ja:新入生代表の挨拶も絵奈だったから目立ってしまって余計にあの子誰\n",
            "3|punctuation:？\n",
            "4|ja:ってなった\n",
            "5|punctuation:。\n",
            "6|ja:絵奈と私には近所に住む幼稚園の頃から仲の良い幼なじみの澤村　和樹\n",
            "7|punctuation:（\n",
            "8|ja:さわむら　かずき\n",
            "9|punctuation:）\n",
            "10|ja:という同じ中学の同級生でもある男子生徒がいる\n",
            "11|punctuation:。\n",
            "12|ja:絵奈も私もかずくんと呼んでいる\n",
            "13|punctuation:。\n",
            "14|ja:かずくんは小学１年生の頃からサッカーをやっていて\n",
            "15|punctuation:、\n",
            "16|ja:高学年になる頃には県の選抜に選ばれたりと\n",
            "17|punctuation:、\n",
            "18|ja:丸山台中の新入生では期待の星\n",
            "19|punctuation:。\n",
            "20|ja:何でユースとかにいかないで\n",
            "21|punctuation:、\n",
            "22|ja:公立の中学校でサッカー部なのかはわからない\n",
            "23|punctuation:。\n",
            "24|ja:かずくんとは絵奈より私の方が話が合うことが多かった\n",
            "25|punctuation:。\n",
            "26|ja:小学生の頃もよく３人で遊んだりもしたし\n",
            "27|punctuation:、\n",
            "28|ja:中学生になってからも３人で朝一緒に行こうなんて約束していた\n",
            "29|punctuation:。\n",
            "30|ja:迎えた入学式後の初登校日\n",
            "31|punctuation:。「\n",
            "32|ja:忘れ物とかない\n",
            "33|punctuation:？\n",
            "34|ja:気を付けて行くんだよ\n",
            "35|punctuation:。\n",
            "36|ja:急がなくても充分間に合うからね\n",
            "37|punctuation:。」\n",
            "38|ja:私にだけそう言ったママ\n",
            "39|punctuation:。\n",
            "40|ja:絵奈には\n",
            "41|punctuation:「\n",
            "42|ja:いってらっしゃい\n",
            "43|punctuation:」\n",
            "44|ja:だけ\n",
            "45|punctuation:。（\n",
            "46|ja:二人とも一緒に行くのに\n",
            "47|punctuation:、\n",
            "48|ja:私だけ落ち着きないみたいに思われてる\n",
            "49|punctuation:…？）「\n",
            "50|ja:ほら\n",
            "51|punctuation:、\n",
            "52|ja:早く行くよ\n",
            "53|punctuation:。\n",
            "54|ja:ママ\n",
            "55|punctuation:、\n",
            "56|ja:いってきます\n",
            "57|de:♪\n",
            "58|punctuation:」\n",
            "59|ja:私の横で姉の絵奈が私に向かってそう言うと\n",
            "60|punctuation:、\n",
            "61|ja:ママには笑顔で挨拶をしていた\n",
            "62|punctuation:。\n",
            "63|ja:そそくさと出てしまう絵奈\n",
            "64|punctuation:。「\n",
            "65|ja:ちょ\n",
            "66|punctuation:、\n",
            "67|ja:ちょっと\n",
            "68|punctuation:（\n",
            "69|x:汗\n",
            "70|punctuation:）\n",
            "71|ja:充分間に合うからねってママ言ってるじゃん\n",
            "72|punctuation:（\n",
            "73|x:汗\n",
            "74|punctuation:）\n",
            "75|ja:んじゃいってきま\n",
            "76|punctuation:〜\n",
            "77|ja:す\n",
            "78|punctuation:！」\n",
            "79|ja:そう言って\n",
            "80|punctuation:、\n",
            "81|ja:私も絵奈の後を急いで追う\n",
            "82|punctuation:。\n",
            "83|ja:外に出るとかずくんがちょっと遠くから歩いてきているのが見えたので\n",
            "84|punctuation:、\n",
            "85|ja:私は大声で手を振って呼んだ\n",
            "86|punctuation:。「\n",
            "87|ja:かずく\n",
            "88|punctuation:〜\n",
            "89|ja:ん\n",
            "90|punctuation:！！\n",
            "91|ja:おはよー\n",
            "92|punctuation:！！」\n",
            "93|ja:私の大声に気づいたかずくんは走ってきた\n",
            "94|punctuation:。「\n",
            "95|ja:よぉ\n",
            "96|punctuation:！！\n",
            "97|ja:絵奈と残念な方の妹\n",
            "98|punctuation:！（\n",
            "99|ja:笑\n",
            "100|punctuation:）」（\n",
            "101|ja:こいつ\n",
            "102|punctuation:…\n",
            "103|ja:面白がってやがるな\n",
            "104|punctuation:…）\n",
            "105|ja:絵奈が新入生代表で目立ってしまって\n",
            "106|punctuation:、\n",
            "107|ja:私は自分のクラスで残念な方の妹だと\n",
            "108|punctuation:、\n",
            "109|ja:昨日\n",
            "110|punctuation:、\n",
            "111|ja:ふざけた男子に言われたのだ\n",
            "112|punctuation:…\n",
            "113|ja:そしてかずくんと私は同じクラスなのでそれを面白がって言ってきたのだろう\n",
            "114|punctuation:…\n",
            "115|ja:うるせぇーよって言ってやろうと思ったら\n",
            "116|punctuation:、\n",
            "117|ja:私より先に絵奈が口を開いた\n",
            "118|punctuation:。「\n",
            "119|ja:かずくん\n",
            "120|punctuation:、\n",
            "121|ja:残念な方の妹って何\n",
            "122|punctuation:？」\n",
            "123|ja:絵奈に聞かれるとちょっとバツが悪そうなかずくんが渋々答える\n",
            "124|punctuation:。「\n",
            "125|ja:いや\n",
            "126|punctuation:、\n",
            "127|ja:その\n",
            "128|punctuation:、\n",
            "129|ja:実は俺達のクラスで\n",
            "130|punctuation:、\n",
            "131|ja:ふざけて紗羅が残念な方の妹って男子にからかわれてたんだよ\n",
            "132|punctuation:（\n",
            "133|x:汗\n",
            "134|punctuation:）」「\n",
            "135|ja:は\n",
            "136|punctuation:？\n",
            "137|ja:で\n",
            "138|punctuation:、\n",
            "139|ja:かずくんも面白がって\n",
            "140|punctuation:、\n",
            "141|ja:そんなこと言ってるわけ\n",
            "142|punctuation:？\n",
            "143|ja:くだらないことやめなよ\n",
            "144|punctuation:。\n",
            "145|ja:そういう時は普通\n",
            "146|punctuation:、\n",
            "147|ja:助ける男がカッコいいんでしょ\n",
            "148|punctuation:。\n",
            "149|ja:一緒になって紗羅が気にしてないからって最悪だよ\n",
            "150|punctuation:。」\n",
            "151|ja:絵奈はどちらかと言うと温厚で優しいタイプなのに少し怒っているように言ったので\n",
            "152|punctuation:、\n",
            "153|ja:かずくんはしょんぼり\n",
            "154|punctuation:、\n",
            "155|ja:私も珍しく絵奈が怒っているので少しびっくりした\n",
            "156|punctuation:…３\n",
            "157|ja:人でちょっと気まずくなって歩くのに耐えられない私が絵奈に言った\n",
            "158|punctuation:。「\n",
            "159|ja:まぁまぁ\n",
            "160|punctuation:、\n",
            "161|ja:そんなツンケンしないでさ\n",
            "162|punctuation:（\n",
            "163|x:汗\n",
            "164|punctuation:）\n",
            "165|ja:絵奈らしくないよ\n",
            "166|punctuation:？\n",
            "167|ja:かずくんも悪気があって言ったわけじゃないんだし\n",
            "168|punctuation:、\n",
            "169|ja:許してあげなよ\n",
            "170|punctuation:（\n",
            "171|ja:笑\n",
            "172|punctuation:）」「\n",
            "173|ja:私らしいって何\n",
            "174|punctuation:？\n",
            "175|ja:紗羅は悔しくないの\n",
            "176|punctuation:？\n",
            "177|ja:残念な方の妹とか言われて\n",
            "178|punctuation:。\n",
            "179|ja:そもそも紗羅がそんなお気楽な感じだから駄目なんでしょ\n",
            "180|punctuation:（\n",
            "181|x:怒\n",
            "182|punctuation:）\n",
            "183|ja:私は紗羅が悪口言われてたら自分のことのように面白くないよ\n",
            "184|punctuation:（\n",
            "185|x:怒\n",
            "186|punctuation:）」（\n",
            "187|ja:怒りの矛先が私に向き始めたな\n",
            "188|punctuation:…）\n",
            "189|ja:絵奈は温厚で優しく優等生タイプだけど\n",
            "190|punctuation:、\n",
            "191|ja:怒るとちょっと面倒くさいタイプ\n",
            "192|punctuation:…\n",
            "193|ja:何日も根に持つし\n",
            "194|punctuation:、\n",
            "195|ja:頑固なところもある\n",
            "196|punctuation:。\n",
            "197|ja:私は対照的でカッとなっても\n",
            "198|punctuation:、１\n",
            "199|ja:日経てばそんなこと忘れてるし\n",
            "200|punctuation:、\n",
            "201|ja:あんまり根に持つこともない\n",
            "202|punctuation:。\n",
            "203|ja:頑固な所は同じかもしれないけど\n",
            "204|punctuation:。\n",
            "205|ja:怒り出すと絵奈をなだめるのは結構大変\n",
            "206|punctuation:…\n",
            "207|ja:普段あんまり怒らないタイプだから\n",
            "208|punctuation:、\n",
            "209|ja:かもしれないけど\n",
            "210|punctuation:…「\n",
            "211|ja:そのさ\n",
            "212|punctuation:、\n",
            "213|ja:絵奈は私と違っておとなしめな品のあるタイプなんだから\n",
            "214|punctuation:、\n",
            "215|ja:あんまりカッカしてるとイメージ崩れるよ\n",
            "216|punctuation:？\n",
            "217|ja:なんちゃって\n",
            "218|punctuation:（\n",
            "219|ja:笑\n",
            "220|punctuation:）\n",
            "221|ja:かずくんも少しびっくりしてるでしょ\n",
            "222|punctuation:（\n",
            "223|x:汗\n",
            "224|punctuation:）\n",
            "225|ja:ほらほら\n",
            "226|punctuation:、\n",
            "227|ja:せっかく入学して登校初日なのに空気悪いのおしまいにして楽しく行こうよ\n",
            "228|de:♪\n",
            "229|punctuation:（\n",
            "230|ja:笑\n",
            "231|punctuation:）」\n",
            "232|ja:一応\n",
            "233|punctuation:、\n",
            "234|ja:私がそう言うとカッカしてた絵奈も機嫌を少し直してくれて\n",
            "235|punctuation:、\n",
            "236|ja:かずくんもごめんって謝って一件落着し\n",
            "237|punctuation:、\n",
            "238|ja:その後は仲良く登校し中学校へ無事に到着\n",
            "239|punctuation:。「\n",
            "240|ja:んじゃまた帰りね\n",
            "241|punctuation:〜\n",
            "242|de:♪\n",
            "243|punctuation:」\n",
            "244|ja:そう言って\n",
            "245|punctuation:、\n",
            "246|ja:絵奈とは別々の教室へ\n",
            "247|punctuation:。\n",
            "248|ja:私はかずくんと同じクラスへ入る\n",
            "249|punctuation:。\n",
            "250|ja:先生が来て\n",
            "251|punctuation:、\n",
            "252|ja:授業が始まる\n",
            "253|punctuation:。\n",
            "254|ja:勉強は嫌いだ\n",
            "255|punctuation:…（\n",
            "256|ja:今日の給食なんだっけかなぁ\n",
            "257|punctuation:〜）\n",
            "258|ja:なんて授業に集中していないと見事に先生に当てられているのに気づかない\n",
            "259|punctuation:。\n",
            "260|ja:隣の席のかずくんが小さな声で\n",
            "261|punctuation:「\n",
            "262|ja:おーい\n",
            "263|punctuation:。\n",
            "264|ja:紗羅\n",
            "265|punctuation:。\n",
            "266|ja:おーい\n",
            "267|punctuation:」\n",
            "268|ja:って言っているのに気づく\n",
            "269|punctuation:。（\n",
            "270|ja:ん\n",
            "271|punctuation:？\n",
            "272|zh:何\n",
            "273|punctuation:？\n",
            "274|ja:教科書忘れたのかこいつは本当にサッカーしかできない奴だからなぁ\n",
            "275|punctuation:。\n",
            "276|ja:しょうがないなぁ\n",
            "277|punctuation:）\n",
            "278|ja:机をくっつけて教科書を見せてやろうとするとかずくんはオドオドする\n",
            "279|punctuation:。「\n",
            "280|ja:ちげーよ\n",
            "281|punctuation:（\n",
            "282|x:汗\n",
            "283|punctuation:）\n",
            "284|ja:紗羅\n",
            "285|punctuation:（\n",
            "286|x:汗\n",
            "287|punctuation:）」\n",
            "288|ja:また小さな声でかずくんが言った後には怒った先生が私の目の前にいた\n",
            "289|punctuation:…「\n",
            "290|ja:山本\n",
            "291|punctuation:、\n",
            "292|ja:お前何やってんだ\n",
            "293|punctuation:？\n",
            "294|ja:あくびしたり\n",
            "295|punctuation:、\n",
            "296|ja:外をボケーっと見てたり\n",
            "297|punctuation:。\n",
            "298|ja:同じ山本のお前の姉とは大違いだな\n",
            "299|punctuation:。」\n",
            "300|ja:先生は不機嫌そうに言う\n",
            "301|punctuation:。（\n",
            "302|ja:すいませんね\n",
            "303|punctuation:〜\n",
            "304|ja:絵奈だったら\n",
            "305|punctuation:、\n",
            "306|ja:ちゃんと授業聞いてるんでしょうね\n",
            "307|punctuation:〜\n",
            "308|ja:私は残念な方の妹なので\n",
            "309|punctuation:（\n",
            "310|ja:笑\n",
            "311|punctuation:））\n",
            "312|ja:なんて怒られているはずなのに思っていると\n",
            "313|punctuation:、\n",
            "314|ja:朝\n",
            "315|punctuation:、\n",
            "316|ja:絵奈から怒られたからか\n",
            "317|punctuation:、\n",
            "318|ja:かずくんが私を庇った\n",
            "319|punctuation:。「\n",
            "320|ja:その\n",
            "321|punctuation:、\n",
            "322|zh:先生\n",
            "323|punctuation:。\n",
            "324|ja:絵奈は絵奈\n",
            "325|punctuation:。\n",
            "326|ja:紗羅は紗羅ですよ\n",
            "327|punctuation:。」\n",
            "328|ja:かずくんの言葉に先生もちょっと悪いと思ったのか\n",
            "329|punctuation:、\n",
            "330|ja:ブツブツ言いながら戻っていき\n",
            "331|punctuation:、\n",
            "332|ja:私がボケーっとしていたことは流されて終わった\n",
            "333|punctuation:。（\n",
            "334|ja:おぉーかずくんが助けてくれた\n",
            "335|punctuation:（\n",
            "336|ja:笑\n",
            "337|punctuation:）\n",
            "338|ja:もしかして\n",
            "339|punctuation:、\n",
            "340|ja:絵奈に朝怒られたから\n",
            "341|punctuation:？\n",
            "342|ja:もしかして\n",
            "343|punctuation:、\n",
            "344|ja:かずくんも昔から絵奈のことが\n",
            "345|punctuation:…\n",
            "346|ja:いや\n",
            "347|punctuation:、\n",
            "348|ja:考えるのはやめよう\n",
            "349|punctuation:。\n",
            "350|ja:私達３人は幼馴染\n",
            "351|punctuation:。\n",
            "352|ja:変に関係がおかしくなるのも嫌だし\n",
            "353|punctuation:。）\n",
            "354|ja:なんてふと思った\n",
            "355|punctuation:。\n",
            "356|ja:そして\n",
            "357|punctuation:、\n",
            "358|ja:この先\n",
            "359|punctuation:、\n",
            "360|ja:私達３人の関係は壊れていくことにこの時は気づかなかった\n",
            "361|punctuation:…\n",
            "----------------------\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.DIGIT\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.OTHERS\n",
            "0|punctuation:2.\n",
            "1|zh:术语和定义 2\n",
            "2|punctuation:.\n",
            "3|en:Terms and Definitions\n",
            "----------------------\n",
            "测试：LangSectionType.DIGIT\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.DIGIT\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.DIGIT\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.DIGIT\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.OTHERS\n",
            "0|punctuation:(2.2)\n",
            "1|zh:术语和定义 \n",
            "2|punctuation:(2.2)\n",
            "3|en:Terms and Definitions\n",
            "----------------------\n",
            "0.1461777687072754\n"
          ]
        }
      ],
      "source": [
        "import time\n",
        "\n",
        "lang_splitter = LangSplitter(merge_across_punctuation=False, merge_across_digit=True)\n",
        "\n",
        "texts = [\n",
        "    \"你喜欢看アニメ吗？我也喜欢看\",\n",
        "    \"Please star this project on GitHub, Thanks you. I love you请加星这个项目，谢谢你。我爱你この項目をスターしてください、ありがとうございます！愛してる\",\n",
        "    \"日本語（にほんご、にっぽんご）は、日本国内や、かつての日本領だった国、そして国外移民や移住者を含む日本人同士の間で使用されている言語。日本は法令によって公用語を規定していないが、法令その他の公用文は全て日本語で記述され、各種法令において日本語を用いることが規定され、学校教育においては「国語」の教科として学習を行うなど、事実上日本国内において唯一の公用語となっている。\",\n",
        "    \"日语是日本通用语及事实上的官方语言。没有精确的日语使用人口的统计，如果计算日本人口以及居住在日本以外的日本人、日侨和日裔，日语使用者应超过一亿三千万人。\",\n",
        "    \"入学早々隣のクラスの絵奈は男子からの人気者。新入生代表の挨拶も絵奈だったから目立ってしまって余計にあの子誰？ってなった。絵奈と私には近所に住む幼稚園の頃から仲の良い幼なじみの澤村　和樹（さわむら　かずき）という同じ中学の同級生でもある男子生徒がいる。絵奈も私もかずくんと呼んでいる。かずくんは小学１年生の頃からサッカーをやっていて、高学年になる頃には県の選抜に選ばれたりと、丸山台中の新入生では期待の星。何でユースとかにいかないで、公立の中学校でサッカー部なのかはわからない。かずくんとは絵奈より私の方が話が合うことが多かった。小学生の頃もよく３人で遊んだりもしたし、中学生になってからも３人で朝一緒に行こうなんて約束していた。迎えた入学式後の初登校日。「忘れ物とかない？気を付けて行くんだよ。急がなくても充分間に合うからね。」私にだけそう言ったママ。絵奈には「いってらっしゃい」だけ。（二人とも一緒に行くのに、私だけ落ち着きないみたいに思われてる…？）「ほら、早く行くよ。ママ、いってきます♪」私の横で姉の絵奈が私に向かってそう言うと、ママには笑顔で挨拶をしていた。そそくさと出てしまう絵奈。「ちょ、ちょっと（汗）充分間に合うからねってママ言ってるじゃん（汗）んじゃいってきま〜す！」そう言って、私も絵奈の後を急いで追う。外に出るとかずくんがちょっと遠くから歩いてきているのが見えたので、私は大声で手を振って呼んだ。「かずく〜ん！！おはよー！！」私の大声に気づいたかずくんは走ってきた。「よぉ！！絵奈と残念な方の妹！（笑）」（こいつ…面白がってやがるな…）絵奈が新入生代表で目立ってしまって、私は自分のクラスで残念な方の妹だと、昨日、ふざけた男子に言われたのだ…そしてかずくんと私は同じクラスなのでそれを面白がって言ってきたのだろう…うるせぇーよって言ってやろうと思ったら、私より先に絵奈が口を開いた。「かずくん、残念な方の妹って何？」絵奈に聞かれるとちょっとバツが悪そうなかずくんが渋々答える。「いや、その、実は俺達のクラスで、ふざけて紗羅が残念な方の妹って男子にからかわれてたんだよ（汗）」「は？で、かずくんも面白がって、そんなこと言ってるわけ？くだらないことやめなよ。そういう時は普通、助ける男がカッコいいんでしょ。一緒になって紗羅が気にしてないからって最悪だよ。」絵奈はどちらかと言うと温厚で優しいタイプなのに少し怒っているように言ったので、かずくんはしょんぼり、私も珍しく絵奈が怒っているので少しびっくりした…３人でちょっと気まずくなって歩くのに耐えられない私が絵奈に言った。「まぁまぁ、そんなツンケンしないでさ（汗）絵奈らしくないよ？かずくんも悪気があって言ったわけじゃないんだし、許してあげなよ（笑）」「私らしいって何？紗羅は悔しくないの？残念な方の妹とか言われて。そもそも紗羅がそんなお気楽な感じだから駄目なんでしょ（怒）私は紗羅が悪口言われてたら自分のことのように面白くないよ（怒）」（怒りの矛先が私に向き始めたな…）絵奈は温厚で優しく優等生タイプだけど、怒るとちょっと面倒くさいタイプ…何日も根に持つし、頑固なところもある。私は対照的でカッとなっても、１日経てばそんなこと忘れてるし、あんまり根に持つこともない。頑固な所は同じかもしれないけど。怒り出すと絵奈をなだめるのは結構大変…普段あんまり怒らないタイプだから、かもしれないけど…「そのさ、絵奈は私と違っておとなしめな品のあるタイプなんだから、あんまりカッカしてるとイメージ崩れるよ？なんちゃって（笑）かずくんも少しびっくりしてるでしょ（汗）ほらほら、せっかく入学して登校初日なのに空気悪いのおしまいにして楽しく行こうよ♪（笑）」一応、私がそう言うとカッカしてた絵奈も機嫌を少し直してくれて、かずくんもごめんって謝って一件落着し、その後は仲良く登校し中学校へ無事に到着。「んじゃまた帰りね〜♪」そう言って、絵奈とは別々の教室へ。私はかずくんと同じクラスへ入る。先生が来て、授業が始まる。勉強は嫌いだ…（今日の給食なんだっけかなぁ〜）なんて授業に集中していないと見事に先生に当てられているのに気づかない。隣の席のかずくんが小さな声で「おーい。紗羅。おーい」って言っているのに気づく。（ん？何？教科書忘れたのかこいつは本当にサッカーしかできない奴だからなぁ。しょうがないなぁ）机をくっつけて教科書を見せてやろうとするとかずくんはオドオドする。「ちげーよ（汗）紗羅（汗）」また小さな声でかずくんが言った後には怒った先生が私の目の前にいた…「山本、お前何やってんだ？あくびしたり、外をボケーっと見てたり。同じ山本のお前の姉とは大違いだな。」先生は不機嫌そうに言う。（すいませんね〜絵奈だったら、ちゃんと授業聞いてるんでしょうね〜私は残念な方の妹なので（笑））なんて怒られているはずなのに思っていると、朝、絵奈から怒られたからか、かずくんが私を庇った。「その、先生。絵奈は絵奈。紗羅は紗羅ですよ。」かずくんの言葉に先生もちょっと悪いと思ったのか、ブツブツ言いながら戻っていき、私がボケーっとしていたことは流されて終わった。（おぉーかずくんが助けてくれた（笑）もしかして、絵奈に朝怒られたから？もしかして、かずくんも昔から絵奈のことが…いや、考えるのはやめよう。私達３人は幼馴染。変に関係がおかしくなるのも嫌だし。）なんてふと思った。そして、この先、私達３人の関係は壊れていくことにこの時は気づかなかった…\",\n",
        "    \"2.术语和定义 2.Terms and Definitions\",\n",
        "    \"(2.2)术语和定义 (2.2)Terms and Definitions\",\n",
        "]\n",
        "time1 = time.time()\n",
        "for text in texts:\n",
        "    substr = lang_splitter.split_by_lang(\n",
        "        text=text,\n",
        "    )\n",
        "    for index, item in enumerate(substr):\n",
        "        print(f\"{index}|{item.lang}:{item.text}\")\n",
        "    print(\"----------------------\")\n",
        "time2 = time.time()\n",
        "print(time2 - time1)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 10,
      "metadata": {},
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "0|zh:你喜欢看\n",
            "1|ja:アニメ\n",
            "2|zh:吗\n",
            "3|punctuation:？\n",
            "4|zh:我也喜欢看\n",
            "----------------------\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.OTHERS\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.OTHERS\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "0|en:Please star this project on GitHub\n",
            "1|punctuation:, \n",
            "2|en:Thanks you\n",
            "3|punctuation:. \n",
            "4|en:I love you\n",
            "5|zh:请加星这个项目\n",
            "6|punctuation:，\n",
            "7|zh:谢谢你\n",
            "8|punctuation:。\n",
            "9|zh:我爱你\n",
            "10|ja:この項目をスターしてください\n",
            "11|punctuation:、\n",
            "12|ja:ありがとうございます\n",
            "13|punctuation:！\n",
            "14|ja:愛してる\n",
            "----------------------\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "0|ja:日本語\n",
            "1|punctuation:（\n",
            "2|ja:にほんご\n",
            "3|punctuation:、\n",
            "4|ja:にっぽんご\n",
            "5|punctuation:）\n",
            "6|ja:は\n",
            "7|punctuation:、\n",
            "8|ja:日本国内や\n",
            "9|punctuation:、\n",
            "10|ja:かつての日本領だった国\n",
            "11|punctuation:、\n",
            "12|ja:そして国外移民や移住者を含む日本人同士の間で使用されている言語\n",
            "13|punctuation:。\n",
            "14|ja:日本は法令によって公用語を規定していないが\n",
            "15|punctuation:、\n",
            "16|ja:法令その他の公用文は全て日本語で記述され\n",
            "17|punctuation:、\n",
            "18|ja:各種法令において日本語を用いることが規定され\n",
            "19|punctuation:、\n",
            "20|ja:学校教育においては\n",
            "21|punctuation:「\n",
            "22|ja:国語\n",
            "23|punctuation:」\n",
            "24|ja:の教科として学習を行うなど\n",
            "25|punctuation:、\n",
            "26|ja:事実上日本国内において唯一の公用語となっている\n",
            "27|punctuation:。\n",
            "----------------------\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "0|zh:日语是日本通用语及事实上的官方语言\n",
            "1|punctuation:。\n",
            "2|zh:没有精确的日语使用人口的统计\n",
            "3|punctuation:，\n",
            "4|zh:如果计算日本人口以及居住在日本以外的日本人\n",
            "5|punctuation:、\n",
            "6|zh:日侨和日裔\n",
            "7|punctuation:，\n",
            "8|zh:日语使用者应超过一亿三千万人\n",
            "9|punctuation:。\n",
            "----------------------\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.DIGIT\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.DIGIT\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.DIGIT\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.OTHERS\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.DIGIT\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.DIGIT\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.OTHERS\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.OTHERS\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.DIGIT\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.DIGIT\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "0|ja:入学早々隣のクラスの絵奈は男子からの人気者\n",
            "1|punctuation:。\n",
            "2|ja:新入生代表の挨拶も絵奈だったから目立ってしまって余計にあの子誰\n",
            "3|punctuation:？\n",
            "4|ja:ってなった\n",
            "5|punctuation:。\n",
            "6|ja:絵奈と私には近所に住む幼稚園の頃から仲の良い幼なじみの澤村　和樹\n",
            "7|punctuation:（\n",
            "8|ja:さわむら　かずき\n",
            "9|punctuation:）\n",
            "10|ja:という同じ中学の同級生でもある男子生徒がいる\n",
            "11|punctuation:。\n",
            "12|ja:絵奈も私もかずくんと呼んでいる\n",
            "13|punctuation:。\n",
            "14|ja:かずくんは小学１年生の頃からサッカーをやっていて\n",
            "15|punctuation:、\n",
            "16|ja:高学年になる頃には県の選抜に選ばれたりと\n",
            "17|punctuation:、\n",
            "18|ja:丸山台中の新入生では期待の星\n",
            "19|punctuation:。\n",
            "20|ja:何でユースとかにいかないで\n",
            "21|punctuation:、\n",
            "22|ja:公立の中学校でサッカー部なのかはわからない\n",
            "23|punctuation:。\n",
            "24|ja:かずくんとは絵奈より私の方が話が合うことが多かった\n",
            "25|punctuation:。\n",
            "26|ja:小学生の頃もよく３人で遊んだりもしたし\n",
            "27|punctuation:、\n",
            "28|ja:中学生になってからも３人で朝一緒に行こうなんて約束していた\n",
            "29|punctuation:。\n",
            "30|ja:迎えた入学式後の初登校日\n",
            "31|punctuation:。「\n",
            "32|ja:忘れ物とかない\n",
            "33|punctuation:？\n",
            "34|ja:気を付けて行くんだよ\n",
            "35|punctuation:。\n",
            "36|ja:急がなくても充分間に合うからね\n",
            "37|punctuation:。」\n",
            "38|ja:私にだけそう言ったママ\n",
            "39|punctuation:。\n",
            "40|ja:絵奈には\n",
            "41|punctuation:「\n",
            "42|ja:いってらっしゃい\n",
            "43|punctuation:」\n",
            "44|ja:だけ\n",
            "45|punctuation:。（\n",
            "46|ja:二人とも一緒に行くのに\n",
            "47|punctuation:、\n",
            "48|ja:私だけ落ち着きないみたいに思われてる\n",
            "49|punctuation:…？）「\n",
            "50|ja:ほら\n",
            "51|punctuation:、\n",
            "52|ja:早く行くよ\n",
            "53|punctuation:。\n",
            "54|ja:ママ\n",
            "55|punctuation:、\n",
            "56|ja:いってきます\n",
            "57|de:♪\n",
            "58|punctuation:」\n",
            "59|ja:私の横で姉の絵奈が私に向かってそう言うと\n",
            "60|punctuation:、\n",
            "61|ja:ママには笑顔で挨拶をしていた\n",
            "62|punctuation:。\n",
            "63|ja:そそくさと出てしまう絵奈\n",
            "64|punctuation:。「\n",
            "65|ja:ちょ\n",
            "66|punctuation:、\n",
            "67|ja:ちょっと\n",
            "68|punctuation:（\n",
            "69|x:汗\n",
            "70|punctuation:）\n",
            "71|ja:充分間に合うからねってママ言ってるじゃん\n",
            "72|punctuation:（\n",
            "73|x:汗\n",
            "74|punctuation:）\n",
            "75|ja:んじゃいってきま\n",
            "76|punctuation:〜\n",
            "77|ja:す\n",
            "78|punctuation:！」\n",
            "79|ja:そう言って\n",
            "80|punctuation:、\n",
            "81|ja:私も絵奈の後を急いで追う\n",
            "82|punctuation:。\n",
            "83|ja:外に出るとかずくんがちょっと遠くから歩いてきているのが見えたので\n",
            "84|punctuation:、\n",
            "85|ja:私は大声で手を振って呼んだ\n",
            "86|punctuation:。「\n",
            "87|ja:かずく\n",
            "88|punctuation:〜\n",
            "89|ja:ん\n",
            "90|punctuation:！！\n",
            "91|ja:おはよー\n",
            "92|punctuation:！！」\n",
            "93|ja:私の大声に気づいたかずくんは走ってきた\n",
            "94|punctuation:。「\n",
            "95|ja:よぉ\n",
            "96|punctuation:！！\n",
            "97|ja:絵奈と残念な方の妹\n",
            "98|punctuation:！（\n",
            "99|ja:笑\n",
            "100|punctuation:）」（\n",
            "101|ja:こいつ\n",
            "102|punctuation:…\n",
            "103|ja:面白がってやがるな\n",
            "104|punctuation:…）\n",
            "105|ja:絵奈が新入生代表で目立ってしまって\n",
            "106|punctuation:、\n",
            "107|ja:私は自分のクラスで残念な方の妹だと\n",
            "108|punctuation:、\n",
            "109|ja:昨日\n",
            "110|punctuation:、\n",
            "111|ja:ふざけた男子に言われたのだ\n",
            "112|punctuation:…\n",
            "113|ja:そしてかずくんと私は同じクラスなのでそれを面白がって言ってきたのだろう\n",
            "114|punctuation:…\n",
            "115|ja:うるせぇーよって言ってやろうと思ったら\n",
            "116|punctuation:、\n",
            "117|ja:私より先に絵奈が口を開いた\n",
            "118|punctuation:。「\n",
            "119|ja:かずくん\n",
            "120|punctuation:、\n",
            "121|ja:残念な方の妹って何\n",
            "122|punctuation:？」\n",
            "123|ja:絵奈に聞かれるとちょっとバツが悪そうなかずくんが渋々答える\n",
            "124|punctuation:。「\n",
            "125|ja:いや\n",
            "126|punctuation:、\n",
            "127|ja:その\n",
            "128|punctuation:、\n",
            "129|ja:実は俺達のクラスで\n",
            "130|punctuation:、\n",
            "131|ja:ふざけて紗羅が残念な方の妹って男子にからかわれてたんだよ\n",
            "132|punctuation:（\n",
            "133|x:汗\n",
            "134|punctuation:）」「\n",
            "135|ja:は\n",
            "136|punctuation:？\n",
            "137|ja:で\n",
            "138|punctuation:、\n",
            "139|ja:かずくんも面白がって\n",
            "140|punctuation:、\n",
            "141|ja:そんなこと言ってるわけ\n",
            "142|punctuation:？\n",
            "143|ja:くだらないことやめなよ\n",
            "144|punctuation:。\n",
            "145|ja:そういう時は普通\n",
            "146|punctuation:、\n",
            "147|ja:助ける男がカッコいいんでしょ\n",
            "148|punctuation:。\n",
            "149|ja:一緒になって紗羅が気にしてないからって最悪だよ\n",
            "150|punctuation:。」\n",
            "151|ja:絵奈はどちらかと言うと温厚で優しいタイプなのに少し怒っているように言ったので\n",
            "152|punctuation:、\n",
            "153|ja:かずくんはしょんぼり\n",
            "154|punctuation:、\n",
            "155|ja:私も珍しく絵奈が怒っているので少しびっくりした\n",
            "156|punctuation:…３\n",
            "157|ja:人でちょっと気まずくなって歩くのに耐えられない私が絵奈に言った\n",
            "158|punctuation:。「\n",
            "159|ja:まぁまぁ\n",
            "160|punctuation:、\n",
            "161|ja:そんなツンケンしないでさ\n",
            "162|punctuation:（\n",
            "163|x:汗\n",
            "164|punctuation:）\n",
            "165|ja:絵奈らしくないよ\n",
            "166|punctuation:？\n",
            "167|ja:かずくんも悪気があって言ったわけじゃないんだし\n",
            "168|punctuation:、\n",
            "169|ja:許してあげなよ\n",
            "170|punctuation:（\n",
            "171|ja:笑\n",
            "172|punctuation:）」「\n",
            "173|ja:私らしいって何\n",
            "174|punctuation:？\n",
            "175|ja:紗羅は悔しくないの\n",
            "176|punctuation:？\n",
            "177|ja:残念な方の妹とか言われて\n",
            "178|punctuation:。\n",
            "179|ja:そもそも紗羅がそんなお気楽な感じだから駄目なんでしょ\n",
            "180|punctuation:（\n",
            "181|x:怒\n",
            "182|punctuation:）\n",
            "183|ja:私は紗羅が悪口言われてたら自分のことのように面白くないよ\n",
            "184|punctuation:（\n",
            "185|x:怒\n",
            "186|punctuation:）」（\n",
            "187|ja:怒りの矛先が私に向き始めたな\n",
            "188|punctuation:…）\n",
            "189|ja:絵奈は温厚で優しく優等生タイプだけど\n",
            "190|punctuation:、\n",
            "191|ja:怒るとちょっと面倒くさいタイプ\n",
            "192|punctuation:…\n",
            "193|ja:何日も根に持つし\n",
            "194|punctuation:、\n",
            "195|ja:頑固なところもある\n",
            "196|punctuation:。\n",
            "197|ja:私は対照的でカッとなっても\n",
            "198|punctuation:、１\n",
            "199|ja:日経てばそんなこと忘れてるし\n",
            "200|punctuation:、\n",
            "201|ja:あんまり根に持つこともない\n",
            "202|punctuation:。\n",
            "203|ja:頑固な所は同じかもしれないけど\n",
            "204|punctuation:。\n",
            "205|ja:怒り出すと絵奈をなだめるのは結構大変\n",
            "206|punctuation:…\n",
            "207|ja:普段あんまり怒らないタイプだから\n",
            "208|punctuation:、\n",
            "209|ja:かもしれないけど\n",
            "210|punctuation:…「\n",
            "211|ja:そのさ\n",
            "212|punctuation:、\n",
            "213|ja:絵奈は私と違っておとなしめな品のあるタイプなんだから\n",
            "214|punctuation:、\n",
            "215|ja:あんまりカッカしてるとイメージ崩れるよ\n",
            "216|punctuation:？\n",
            "217|ja:なんちゃって\n",
            "218|punctuation:（\n",
            "219|ja:笑\n",
            "220|punctuation:）\n",
            "221|ja:かずくんも少しびっくりしてるでしょ\n",
            "222|punctuation:（\n",
            "223|x:汗\n",
            "224|punctuation:）\n",
            "225|ja:ほらほら\n",
            "226|punctuation:、\n",
            "227|ja:せっかく入学して登校初日なのに空気悪いのおしまいにして楽しく行こうよ\n",
            "228|de:♪\n",
            "229|punctuation:（\n",
            "230|ja:笑\n",
            "231|punctuation:）」\n",
            "232|ja:一応\n",
            "233|punctuation:、\n",
            "234|ja:私がそう言うとカッカしてた絵奈も機嫌を少し直してくれて\n",
            "235|punctuation:、\n",
            "236|ja:かずくんもごめんって謝って一件落着し\n",
            "237|punctuation:、\n",
            "238|ja:その後は仲良く登校し中学校へ無事に到着\n",
            "239|punctuation:。「\n",
            "240|ja:んじゃまた帰りね\n",
            "241|punctuation:〜\n",
            "242|de:♪\n",
            "243|punctuation:」\n",
            "244|ja:そう言って\n",
            "245|punctuation:、\n",
            "246|ja:絵奈とは別々の教室へ\n",
            "247|punctuation:。\n",
            "248|ja:私はかずくんと同じクラスへ入る\n",
            "249|punctuation:。\n",
            "250|ja:先生が来て\n",
            "251|punctuation:、\n",
            "252|ja:授業が始まる\n",
            "253|punctuation:。\n",
            "254|ja:勉強は嫌いだ\n",
            "255|punctuation:…（\n",
            "256|ja:今日の給食なんだっけかなぁ\n",
            "257|punctuation:〜）\n",
            "258|ja:なんて授業に集中していないと見事に先生に当てられているのに気づかない\n",
            "259|punctuation:。\n",
            "260|ja:隣の席のかずくんが小さな声で\n",
            "261|punctuation:「\n",
            "262|ja:おーい\n",
            "263|punctuation:。\n",
            "264|ja:紗羅\n",
            "265|punctuation:。\n",
            "266|ja:おーい\n",
            "267|punctuation:」\n",
            "268|ja:って言っているのに気づく\n",
            "269|punctuation:。（\n",
            "270|ja:ん\n",
            "271|punctuation:？\n",
            "272|zh:何\n",
            "273|punctuation:？\n",
            "274|ja:教科書忘れたのかこいつは本当にサッカーしかできない奴だからなぁ\n",
            "275|punctuation:。\n",
            "276|ja:しょうがないなぁ\n",
            "277|punctuation:）\n",
            "278|ja:机をくっつけて教科書を見せてやろうとするとかずくんはオドオドする\n",
            "279|punctuation:。「\n",
            "280|ja:ちげーよ\n",
            "281|punctuation:（\n",
            "282|x:汗\n",
            "283|punctuation:）\n",
            "284|ja:紗羅\n",
            "285|punctuation:（\n",
            "286|x:汗\n",
            "287|punctuation:）」\n",
            "288|ja:また小さな声でかずくんが言った後には怒った先生が私の目の前にいた\n",
            "289|punctuation:…「\n",
            "290|ja:山本\n",
            "291|punctuation:、\n",
            "292|ja:お前何やってんだ\n",
            "293|punctuation:？\n",
            "294|ja:あくびしたり\n",
            "295|punctuation:、\n",
            "296|ja:外をボケーっと見てたり\n",
            "297|punctuation:。\n",
            "298|ja:同じ山本のお前の姉とは大違いだな\n",
            "299|punctuation:。」\n",
            "300|ja:先生は不機嫌そうに言う\n",
            "301|punctuation:。（\n",
            "302|ja:すいませんね\n",
            "303|punctuation:〜\n",
            "304|ja:絵奈だったら\n",
            "305|punctuation:、\n",
            "306|ja:ちゃんと授業聞いてるんでしょうね\n",
            "307|punctuation:〜\n",
            "308|ja:私は残念な方の妹なので\n",
            "309|punctuation:（\n",
            "310|ja:笑\n",
            "311|punctuation:））\n",
            "312|ja:なんて怒られているはずなのに思っていると\n",
            "313|punctuation:、\n",
            "314|ja:朝\n",
            "315|punctuation:、\n",
            "316|ja:絵奈から怒られたからか\n",
            "317|punctuation:、\n",
            "318|ja:かずくんが私を庇った\n",
            "319|punctuation:。「\n",
            "320|ja:その\n",
            "321|punctuation:、\n",
            "322|zh:先生\n",
            "323|punctuation:。\n",
            "324|ja:絵奈は絵奈\n",
            "325|punctuation:。\n",
            "326|ja:紗羅は紗羅ですよ\n",
            "327|punctuation:。」\n",
            "328|ja:かずくんの言葉に先生もちょっと悪いと思ったのか\n",
            "329|punctuation:、\n",
            "330|ja:ブツブツ言いながら戻っていき\n",
            "331|punctuation:、\n",
            "332|ja:私がボケーっとしていたことは流されて終わった\n",
            "333|punctuation:。（\n",
            "334|ja:おぉーかずくんが助けてくれた\n",
            "335|punctuation:（\n",
            "336|ja:笑\n",
            "337|punctuation:）\n",
            "338|ja:もしかして\n",
            "339|punctuation:、\n",
            "340|ja:絵奈に朝怒られたから\n",
            "341|punctuation:？\n",
            "342|ja:もしかして\n",
            "343|punctuation:、\n",
            "344|ja:かずくんも昔から絵奈のことが\n",
            "345|punctuation:…\n",
            "346|ja:いや\n",
            "347|punctuation:、\n",
            "348|ja:考えるのはやめよう\n",
            "349|punctuation:。\n",
            "350|ja:私達３人は幼馴染\n",
            "351|punctuation:。\n",
            "352|ja:変に関係がおかしくなるのも嫌だし\n",
            "353|punctuation:。）\n",
            "354|ja:なんてふと思った\n",
            "355|punctuation:。\n",
            "356|ja:そして\n",
            "357|punctuation:、\n",
            "358|ja:この先\n",
            "359|punctuation:、\n",
            "360|ja:私達３人の関係は壊れていくことにこの時は気づかなかった\n",
            "361|punctuation:…\n",
            "----------------------\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.DIGIT\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.OTHERS\n",
            "0|punctuation:2.\n",
            "1|zh:术语和定义 2\n",
            "2|punctuation:.\n",
            "3|en:Terms and Definitions\n",
            "----------------------\n",
            "测试：LangSectionType.DIGIT\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.DIGIT\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.ZH_JA\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.DIGIT\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.DIGIT\n",
            "测试：LangSectionType.PUNCTUATION\n",
            "测试：LangSectionType.OTHERS\n",
            "0|punctuation:(2.2)\n",
            "1|zh:术语和定义 \n",
            "2|punctuation:(2.2)\n",
            "3|en:Terms and Definitions\n",
            "----------------------\n",
            "0.17491483688354492\n"
          ]
        }
      ],
      "source": [
        "lang_splitter.merge_across_punctuation = False\n",
        "time1 = time.time()\n",
        "for text in texts:\n",
        "    substr = lang_splitter.split_by_lang(\n",
        "        text=text,\n",
        "    )\n",
        "    for index, item in enumerate(substr):\n",
        "        print(f\"{index}|{item.lang}:{item.text}\")\n",
        "    print(\"----------------------\")\n",
        "time2 = time.time()\n",
        "print(time2 - time1)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 11,
      "metadata": {},
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "0|zh:衬衫的价格是\n",
            "1|digit:9\n",
            "2|punctuation:.\n",
            "3|digit:15\n",
            "4|zh:便士\n"
          ]
        }
      ],
      "source": [
        "lang_splitter.merge_across_digit = False\n",
        "texts = [\n",
        "    \"衬衫的价格是9.15便士\",\n",
        "]\n",
        "for text in texts:\n",
        "    substr = lang_splitter.split_by_lang(\n",
        "        text=text,\n",
        "    )\n",
        "    for index, item in enumerate(substr):\n",
        "        print(f\"{index}|{item.lang}:{item.text}\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 12,
      "metadata": {},
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "测试：LangSectionType.DIGIT\n",
            "测试：LangSectionType.ZH_JA\n",
            "0|zh:衬衫的价格是9.15便士\n"
          ]
        }
      ],
      "source": [
        "lang_splitter.merge_across_digit = True\n",
        "lang_splitter.merge_across_punctuation = True\n",
        "texts = [\n",
        "    \"衬衫的价格是9.15便士\",\n",
        "]\n",
        "for text in texts:\n",
        "    substr = lang_splitter.split_by_lang(\n",
        "        text=text,\n",
        "    )\n",
        "    for index, item in enumerate(substr):\n",
        "        print(f\"{index}|{item.lang}:{item.text}\")"
      ]
    }
  ],
  "metadata": {
    "kernelspec": {
      "display_name": "melotts",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.10.14"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 2
}