add_thai_token.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "ecad2cb8-d68b-4709-82ea-948afe439733",
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import AutoTokenizer, AutoModelForCausalLM\n",
    "\n",
    "tokenizer = AutoTokenizer.from_pretrained(\"Qwen/Qwen2.5-7B-Instruct\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "fae06eae-5b01-49af-9cbd-97f8955611ae",
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "pat = re.compile('[ก-๙]') # คำไทย คือมีอักษรไทย อยู่ในช่วงประมาณนี้\n",
    "# cr Cr. https://colab.research.google.com/drive/1SdaxK-jOOZDWS-2z_0RoYVuSdnr_Ca6B#scrollTo=fWq7xXOAPz6_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "5c839dff-49c0-4e91-80ad-5b6ec2fd2140",
   "metadata": {},
   "outputs": [],
   "source": [
    "encode = tokenizer.encode\n",
    "decode = tokenizer.decode\n",
    "tokenize = tokenizer.tokenize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "1e4ecb5b-2d02-4071-95f2-91791cae5cf7",
   "metadata": {},
   "outputs": [],
   "source": [
    "vocab=tokenizer.vocab"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "551e4af7-754a-488f-aa13-1758e80e9d81",
   "metadata": {},
   "outputs": [],
   "source": [
    "all_vocabs = []\n",
    "for v in vocab.values():\n",
    "  k = decode(v)\n",
    "  all_vocabs.append(k)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "dd8e4e4e-6990-4ae8-9aa5-9a872f2c9ae5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2569"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "thais = [(i,v) for i,v in enumerate(all_vocabs) if pat.search(v)]\n",
    "len(thais)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "a19ae1b2-5e52-488b-ae04-bce36de5f7a2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(69, 'เกม'),\n",
       " (127, 'นิด'),\n",
       " (130, 'บาง'),\n",
       " (134, 'วิทย'),\n",
       " (150, 'รับรอง'),\n",
       " (151, 'ขึ้น'),\n",
       " (158, 'มหาวิทยา'),\n",
       " (175, 'หนัก'),\n",
       " (213, 'อุ'),\n",
       " (223, 'แค')]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "thais[0:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "d1b0026c-a254-486d-a5c7-0e3efd357ec4",
   "metadata": {},
   "outputs": [],
   "source": [
    "sorted=[]\n",
    "with open(\"rwkv_vocab_v20230424.txt\", \"r\") as f:\n",
    "    for l in f.readlines():\n",
    "        x = eval(l[l.index(' '):l.rindex(' ')])\n",
    "        x = x.encode(\"utf-8\") if isinstance(x, str) else x\n",
    "        sorted += [x]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "ebf5547e-9978-4b49-a67a-d0df6fcbafcf",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "b'Et'"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sorted[1000]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "81ed92af-df83-4add-8c99-c5ffcd3b3c0a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'ที่'"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# latest thai token\n",
    "sorted[58513].decode(\"utf-8\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "27aa3723-c36c-42c8-bf38-6f753a3da1b5",
   "metadata": {},
   "outputs": [],
   "source": [
    "def thai_search(i):\n",
    "    try:\n",
    "        return pat.search(i.decode(\"utf-8\"))\n",
    "    except:\n",
    "        return False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "e852de64-b2f5-4ae7-9363-a6e950bd92ea",
   "metadata": {},
   "outputs": [],
   "source": [
    "rwkv_thais = [i.decode(\"utf-8\") for i in sorted if thai_search(i)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "0b2f3d0b-1a59-498f-97da-7b5ecf8cc9cf",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "59"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(rwkv_thais)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "3bc56f6c-d2f1-4902-ad39-7a7e1cef296d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\"ฅ\" in rwkv_thais"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "85d310d6-6bf7-45c9-a9ed-543f6653e04b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\"๖\" in rwkv_thais"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "7b2fd6bd-4370-4f3a-a6eb-d161d16ddee4",
   "metadata": {},
   "outputs": [],
   "source": [
    "thai_set_rwkv=set(rwkv_thais)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "41362ea4-f620-4b3d-b4d9-c65ffa25eae4",
   "metadata": {},
   "outputs": [],
   "source": [
    "from pythainlp import thai_characters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "8f76682b-a905-4f45-9eed-f90bc78d17ef",
   "metadata": {},
   "outputs": [],
   "source": [
    "add_thai_token=[]\n",
    "for i in thai_characters:\n",
    "    if i not in thai_set_rwkv:\n",
    "        add_thai_token.append(i.encode(\"utf-8\") if isinstance(i, str) else i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "4b0e3edb-5edc-4c6d-91cb-e72414578844",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "40"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(add_thai_token)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "f9193c01-b2c1-4f24-9abe-85c0799bb6e5",
   "metadata": {},
   "outputs": [],
   "source": [
    "for _,i in thais:\n",
    "    if i not in thai_set_rwkv:\n",
    "        add_thai_token.append(i.encode(\"utf-8\") if isinstance(i, str) else i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "5035de14-7560-4eda-a98e-1822c2ffd0ef",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2551"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(add_thai_token)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "2c33d56e-169c-46b4-a23e-285a35d3d4d3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "b'\\xe0\\xb8\\x83'"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "add_thai_token[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "4e0c2029-aaeb-4580-8372-31e6a0a0ae2a",
   "metadata": {},
   "outputs": [],
   "source": [
    "new_sorted=sorted+add_thai_token"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "f96d4394-779e-406f-8189-1472744542b5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "68080"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(new_sorted)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "a0d0f076-7be3-4684-8101-587451baf7d9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[b'\\x00', b'\\x01', b'\\x02', b'\\x03', b'\\x04']"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_sorted[0:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "958b25ac-fc83-4888-8513-e4c1b7148ff9",
   "metadata": {},
   "outputs": [],
   "source": [
    "rwkv_sorted = [(i,len(i)) for  i in new_sorted]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "182e234e-b038-4cdc-8759-d12d9ae5f7f5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(b'\\x00', 1),\n",
       " (b'\\x01', 1),\n",
       " (b'\\x02', 1),\n",
       " (b'\\x03', 1),\n",
       " (b'\\x04', 1),\n",
       " (b'\\x05', 1),\n",
       " (b'\\x06', 1),\n",
       " (b'\\x07', 1),\n",
       " (b'\\x08', 1),\n",
       " (b'\\t', 1)]"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rwkv_sorted[0:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "bbf6e023-f992-4b1a-98f2-a8d9ff1671dc",
   "metadata": {},
   "outputs": [],
   "source": [
    "rwkv_sorted.sort(key=lambda tup: tup[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "51480988-9844-446c-898f-3e2d4cee6b9d",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"rwkv_vocab_v20250216.txt\",\"w\") as f:\n",
    "    for index,(k,l) in enumerate(rwkv_sorted, 1):\n",
    "        f.write(f\"{index} {k} {l}\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "854d02f3-aae2-44d2-a1c4-aa55367c4b09",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}