Pretraining-CNN_Daily.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "057b8705",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import torch \n",
    "import math"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "1f8af1ee",
   "metadata": {},
   "outputs": [],
   "source": [
    "#tokenizer\n",
    "from transformers import BertTokenizer\n",
    "\n",
    "tokenizer = BertTokenizer.from_pretrained(\"bert-base-uncased\", do_basic_tokenization = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "5b9f2a36",
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch.utils.data import Dataset, DataLoader\n",
    "\n",
    "def data_collate(batch_dataset):\n",
    "    arr = np.array(batch_dataset)\n",
    "    inputs = tokenizer(text = arr.tolist(), padding = 'max_length', max_length = 512, truncation=True, return_tensors = 'pt')\n",
    "    return inputs\n",
    "\n",
    "class CreateDataset(Dataset):\n",
    "    def __init__(self, src, tokenizer):\n",
    "        #src = sentences \n",
    "        self.src = src\n",
    "        self.tokenizer = tokenizer\n",
    "        \n",
    "    def __len__(self):\n",
    "        return len(self.src)\n",
    "\n",
    "    def __getitem__(self, idx):\n",
    "        src = self.src[idx]\n",
    "        return src"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "bb78ad04",
   "metadata": {},
   "outputs": [],
   "source": [
    "from datasets import load_dataset\n",
    "\n",
    "data = load_dataset(\"cnn_dailymail\", \"2.0.0\", split = 'train')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "74ecdc8b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'LONDON, England (Reuters) -- Harry Potter star Daniel Radcliffe gains access to a reported £20 million ($41.1 million) fortune as he turns 18 on Monday, but he insists the money won\\'t cast a spell on him. Daniel Radcliffe as Harry Potter in \"Harry Potter and the Order of the Phoenix\" To the disappointment of gossip columnists around the world, the young actor says he has no plans to fritter his cash away on fast cars, drink and celebrity parties. \"I don\\'t plan to be one of those people who, as soon as they turn 18, suddenly buy themselves a massive sports car collection or something similar,\" he told an Australian interviewer earlier this month. \"I don\\'t think I\\'ll be particularly extravagant. \"The things I like buying are things that cost about 10 pounds -- books and CDs and DVDs.\" At 18, Radcliffe will be able to gamble in a casino, buy a drink in a pub or see the horror film \"Hostel: Part II,\" currently six places below his number one movie on the UK box office chart. Details of how he\\'ll mark his landmark birthday are under wraps. His agent and publicist had no comment on his plans. \"I\\'ll definitely have some sort of party,\" he said in an interview. \"Hopefully none of you will be reading about it.\" Radcliffe\\'s earnings from the first five Potter films have been held in a trust fund which he has not been able to touch. Despite his growing fame and riches, the actor says he is keeping his feet firmly on the ground. \"People are always looking to say \\'kid star goes off the rails,\\'\" he told reporters last month. \"But I try very hard not to go that way because it would be too easy for them.\" His latest outing as the boy wizard in \"Harry Potter and the Order of the Phoenix\" is breaking records on both sides of the Atlantic and he will reprise the role in the last two films.  Watch I-Reporter give her review of Potter\\'s latest » . There is life beyond Potter, however. The Londoner has filmed a TV movie called \"My Boy Jack,\" about author Rudyard Kipling and his son, due for release later this year. He will also appear in \"December Boys,\" an Australian film about four boys who escape an orphanage. Earlier this year, he made his stage debut playing a tortured teenager in Peter Shaffer\\'s \"Equus.\" Meanwhile, he is braced for even closer media scrutiny now that he\\'s legally an adult: \"I just think I\\'m going to be more sort of fair game,\" he told Reuters. E-mail to a friend . Copyright 2007 Reuters. All rights reserved.This material may not be published, broadcast, rewritten, or redistributed.'"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data[0]['article']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "436b24a3",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Token indices sequence length is longer than the specified maximum sequence length for this model (587 > 512). Running this sequence through the model will result in indexing errors\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[  101,  2414,  1010,  2563,  1006, 26665,  1007,  1011,  1011,  4302,\n",
      "         10693,  2732,  3817, 22603, 12154,  3229,  2000,  1037,  2988, 21853,\n",
      "          2692,  2454,  1006,  1002,  4601,  1012,  1015,  2454,  1007,  7280,\n",
      "          2004,  2002,  4332,  2324,  2006,  6928,  1010,  2021,  2002, 16818,\n",
      "          1996,  2769,  2180,  1005,  1056,  3459,  1037,  6297,  2006,  2032,\n",
      "          1012,  3817, 22603,  2004,  4302, 10693,  1999,  1000,  4302, 10693,\n",
      "          1998,  1996,  2344,  1997,  1996,  6708,  1000,  2000,  1996, 10520,\n",
      "          1997, 13761, 13317,  2015,  2105,  1996,  2088,  1010,  1996,  2402,\n",
      "          3364,  2758,  2002,  2038,  2053,  3488,  2000, 10424, 27100,  2099,\n",
      "          2010,  5356,  2185,  2006,  3435,  3765,  1010,  4392,  1998,  8958,\n",
      "          4243,  1012,  1000,  1045,  2123,  1005,  1056,  2933,  2000,  2022,\n",
      "          2028,  1997,  2216,  2111,  2040,  1010,  2004,  2574,  2004,  2027,\n",
      "          2735,  2324,  1010,  3402,  4965,  3209,  1037,  5294,  2998,  2482,\n",
      "          3074,  2030,  2242,  2714,  1010,  1000,  2002,  2409,  2019,  2827,\n",
      "          4357,  2121,  3041,  2023,  3204,  1012,  1000,  1045,  2123,  1005,\n",
      "          1056,  2228,  1045,  1005,  2222,  2022,  3391, 27856,  1012,  1000,\n",
      "          1996,  2477,  1045,  2066,  9343,  2024,  2477,  2008,  3465,  2055,\n",
      "          2184,  7038,  1011,  1011,  2808,  1998, 14340,  1998, 22477,  1012,\n",
      "          1000,  2012,  2324,  1010, 22603,  2097,  2022,  2583,  2000, 18503,\n",
      "          1999,  1037,  9270,  1010,  4965,  1037,  4392,  1999,  1037,  9047,\n",
      "          2030,  2156,  1996,  5469,  2143,  1000, 21071,  1024,  2112,  2462,\n",
      "          1010,  1000,  2747,  2416,  3182,  2917,  2010,  2193,  2028,  3185,\n",
      "          2006,  1996,  2866,  3482,  2436,  3673,  1012,  4751,  1997,  2129,\n",
      "          2002,  1005,  2222,  2928,  2010,  8637,  5798,  2024,  2104, 19735,\n",
      "          1012,  2010,  4005,  1998,  2270,  2923,  2018,  2053,  7615,  2006,\n",
      "          2010,  3488,  1012,  1000,  1045,  1005,  2222,  5791,  2031,  2070,\n",
      "          4066,  1997,  2283,  1010,  1000,  2002,  2056,  1999,  2019,  4357,\n",
      "          1012,  1000, 11504,  3904,  1997,  2017,  2097,  2022,  3752,  2055,\n",
      "          2009,  1012,  1000, 22603,  1005,  1055, 16565,  2013,  1996,  2034,\n",
      "          2274, 10693,  3152,  2031,  2042,  2218,  1999,  1037,  3404,  4636,\n",
      "          2029,  2002,  2038,  2025,  2042,  2583,  2000,  3543,  1012,  2750,\n",
      "          2010,  3652,  4476,  1998, 26768,  1010,  1996,  3364,  2758,  2002,\n",
      "          2003,  4363,  2010,  2519,  7933,  2006,  1996,  2598,  1012,  1000,\n",
      "          2111,  2024,  2467,  2559,  2000,  2360,  1005,  4845,  2732,  3632,\n",
      "          2125,  1996, 15168,  1010,  1005,  1000,  2002,  2409, 12060,  2197,\n",
      "          3204,  1012,  1000,  2021,  1045,  3046,  2200,  2524,  2025,  2000,\n",
      "          2175,  2008,  2126,  2138,  2009,  2052,  2022,  2205,  3733,  2005,\n",
      "          2068,  1012,  1000,  2010,  6745, 26256,  2004,  1996,  2879, 10276,\n",
      "          1999,  1000,  4302, 10693,  1998,  1996,  2344,  1997,  1996,  6708,\n",
      "          1000,  2003,  4911,  2636,  2006,  2119,  3903,  1997,  1996,  4448,\n",
      "          1998,  2002,  2097, 16851,  1996,  2535,  1999,  1996,  2197,  2048,\n",
      "          3152,  1012,  3422,  1045,  1011,  6398,  2507,  2014,  3319,  1997,\n",
      "         10693,  1005,  1055,  6745,  1090,  1012,  2045,  2003,  2166,  3458,\n",
      "         10693,  1010,  2174,  1012,  1996,  2414,  2121,  2038,  6361,  1037,\n",
      "          2694,  3185,  2170,  1000,  2026,  2879,  2990,  1010,  1000,  2055,\n",
      "          3166, 18254,  4232, 11382, 14353,  1998,  2010,  2365,  1010,  2349,\n",
      "          2005,  2713,  2101,  2023,  2095,  1012,  2002,  2097,  2036,  3711,\n",
      "          1999,  1000,  2285,  3337,  1010,  1000,  2019,  2827,  2143,  2055,\n",
      "          2176,  3337,  2040,  4019,  2019, 18504,  1012,  3041,  2023,  2095,\n",
      "          1010,  2002,  2081,  2010,  2754,  2834,  2652,  1037, 12364, 10563,\n",
      "          1999,  2848, 21146, 12494,  1005,  1055,  1000,  1041, 28940,  2271,\n",
      "          1012,  1000,  5564,  1010,  2002,  2003, 15515,  2005,  2130,  3553,\n",
      "          2865, 17423,  2085,  2008,  2002,  1005,  1055, 10142,  2019,  4639,\n",
      "          1024,  1000,  1045,  2074,  2228,  1045,  1005,  1049,  2183,  2000,\n",
      "          2022,  2062,  4066,  1997,  4189,  2208,  1010,  1000,  2002,  2409,\n",
      "         26665,  1012,  1041,  1011,  5653,  2000,  1037,  2767,  1012,  9385,\n",
      "          2289, 26665,  1012,  2035,  2916,  9235,  1012,  2023,  3430,  2089,\n",
      "          2025,  2022,  2405,  1010,  3743,  1010,  2128, 15773,  1010,  2030,\n",
      "          2417,  2923,  3089,  8569,  3064,  1012,   102]])\n",
      "587\n"
     ]
    }
   ],
   "source": [
    "#check tokens\n",
    "#max_length = 512 for BERT tokenizer\n",
    "\n",
    "sample_input = np.array(data[0]['article'])\n",
    "tokenizer_output = tokenizer(text = sample_input.tolist(), return_tensors = 'pt')\n",
    "tokens = tokenizer_output['input_ids']\n",
    "\n",
    "print(tokens)\n",
    "print(len(tokens[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "590de7a3",
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "\n",
    "def filter_data(text):\n",
    "    #remove last line\n",
    "    text = re.sub(r\"Copyright \\d{4} Reuters. All rights reserved.*\", \"\", text)\n",
    "    \n",
    "    #replace \\'\n",
    "    text = text.replace(\"\\'\", \"\")\n",
    "    \n",
    "    #replace 's\n",
    "    text = re.sub(r\"'s\\b'\", \"\", text)\n",
    "    \n",
    "    #remove extra white space\n",
    "    text = re.sub(r'\\s+', ' ', text).strip()\n",
    "    \n",
    "    return text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "02c04551",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LONDON, England (Reuters) -- Harry Potter star Daniel Radcliffe gains access to a reported £20 million ($41.1 million) fortune as he turns 18 on Monday, but he insists the money wont cast a spell on him. Daniel Radcliffe as Harry Potter in \"Harry Potter and the Order of the Phoenix\" To the disappointment of gossip columnists around the world, the young actor says he has no plans to fritter his cash away on fast cars, drink and celebrity parties. \"I dont plan to be one of those people who, as soon as they turn 18, suddenly buy themselves a massive sports car collection or something similar,\" he told an Australian interviewer earlier this month. \"I dont think Ill be particularly extravagant. \"The things I like buying are things that cost about 10 pounds -- books and CDs and DVDs.\" At 18, Radcliffe will be able to gamble in a casino, buy a drink in a pub or see the horror film \"Hostel: Part II,\" currently six places below his number one movie on the UK box office chart. Details of how hell mark his landmark birthday are under wraps. His agent and publicist had no comment on his plans. \"Ill definitely have some sort of party,\" he said in an interview. \"Hopefully none of you will be reading about it.\" Radcliffes earnings from the first five Potter films have been held in a trust fund which he has not been able to touch. Despite his growing fame and riches, the actor says he is keeping his feet firmly on the ground. \"People are always looking to say kid star goes off the rails,\" he told reporters last month. \"But I try very hard not to go that way because it would be too easy for them.\" His latest outing as the boy wizard in \"Harry Potter and the Order of the Phoenix\" is breaking records on both sides of the Atlantic and he will reprise the role in the last two films. Watch I-Reporter give her review of Potters latest » . There is life beyond Potter, however. The Londoner has filmed a TV movie called \"My Boy Jack,\" about author Rudyard Kipling and his son, due for release later this year. He will also appear in \"December Boys,\" an Australian film about four boys who escape an orphanage. Earlier this year, he made his stage debut playing a tortured teenager in Peter Shaffers \"Equus.\" Meanwhile, he is braced for even closer media scrutiny now that hes legally an adult: \"I just think Im going to be more sort of fair game,\" he told Reuters. E-mail to a friend .\n",
      "tensor([[  101,  2414,  1010,  2563,  1006, 26665,  1007,  1011,  1011,  4302,\n",
      "         10693,  2732,  3817, 22603, 12154,  3229,  2000,  1037,  2988, 21853,\n",
      "          2692,  2454,  1006,  1002,  4601,  1012,  1015,  2454,  1007,  7280,\n",
      "          2004,  2002,  4332,  2324,  2006,  6928,  1010,  2021,  2002, 16818,\n",
      "          1996,  2769,  2180,  2102,  3459,  1037,  6297,  2006,  2032,  1012,\n",
      "          3817, 22603,  2004,  4302, 10693,  1999,  1000,  4302, 10693,  1998,\n",
      "          1996,  2344,  1997,  1996,  6708,  1000,  2000,  1996, 10520,  1997,\n",
      "         13761, 13317,  2015,  2105,  1996,  2088,  1010,  1996,  2402,  3364,\n",
      "          2758,  2002,  2038,  2053,  3488,  2000, 10424, 27100,  2099,  2010,\n",
      "          5356,  2185,  2006,  3435,  3765,  1010,  4392,  1998,  8958,  4243,\n",
      "          1012,  1000,  1045,  2123,  2102,  2933,  2000,  2022,  2028,  1997,\n",
      "          2216,  2111,  2040,  1010,  2004,  2574,  2004,  2027,  2735,  2324,\n",
      "          1010,  3402,  4965,  3209,  1037,  5294,  2998,  2482,  3074,  2030,\n",
      "          2242,  2714,  1010,  1000,  2002,  2409,  2019,  2827,  4357,  2121,\n",
      "          3041,  2023,  3204,  1012,  1000,  1045,  2123,  2102,  2228,  5665,\n",
      "          2022,  3391, 27856,  1012,  1000,  1996,  2477,  1045,  2066,  9343,\n",
      "          2024,  2477,  2008,  3465,  2055,  2184,  7038,  1011,  1011,  2808,\n",
      "          1998, 14340,  1998, 22477,  1012,  1000,  2012,  2324,  1010, 22603,\n",
      "          2097,  2022,  2583,  2000, 18503,  1999,  1037,  9270,  1010,  4965,\n",
      "          1037,  4392,  1999,  1037,  9047,  2030,  2156,  1996,  5469,  2143,\n",
      "          1000, 21071,  1024,  2112,  2462,  1010,  1000,  2747,  2416,  3182,\n",
      "          2917,  2010,  2193,  2028,  3185,  2006,  1996,  2866,  3482,  2436,\n",
      "          3673,  1012,  4751,  1997,  2129,  3109,  2928,  2010,  8637,  5798,\n",
      "          2024,  2104, 19735,  1012,  2010,  4005,  1998,  2270,  2923,  2018,\n",
      "          2053,  7615,  2006,  2010,  3488,  1012,  1000,  5665,  5791,  2031,\n",
      "          2070,  4066,  1997,  2283,  1010,  1000,  2002,  2056,  1999,  2019,\n",
      "          4357,  1012,  1000, 11504,  3904,  1997,  2017,  2097,  2022,  3752,\n",
      "          2055,  2009,  1012,  1000, 22603,  2015, 16565,  2013,  1996,  2034,\n",
      "          2274, 10693,  3152,  2031,  2042,  2218,  1999,  1037,  3404,  4636,\n",
      "          2029,  2002,  2038,  2025,  2042,  2583,  2000,  3543,  1012,  2750,\n",
      "          2010,  3652,  4476,  1998, 26768,  1010,  1996,  3364,  2758,  2002,\n",
      "          2003,  4363,  2010,  2519,  7933,  2006,  1996,  2598,  1012,  1000,\n",
      "          2111,  2024,  2467,  2559,  2000,  2360,  4845,  2732,  3632,  2125,\n",
      "          1996, 15168,  1010,  1000,  2002,  2409, 12060,  2197,  3204,  1012,\n",
      "          1000,  2021,  1045,  3046,  2200,  2524,  2025,  2000,  2175,  2008,\n",
      "          2126,  2138,  2009,  2052,  2022,  2205,  3733,  2005,  2068,  1012,\n",
      "          1000,  2010,  6745, 26256,  2004,  1996,  2879, 10276,  1999,  1000,\n",
      "          4302, 10693,  1998,  1996,  2344,  1997,  1996,  6708,  1000,  2003,\n",
      "          4911,  2636,  2006,  2119,  3903,  1997,  1996,  4448,  1998,  2002,\n",
      "          2097, 16851,  1996,  2535,  1999,  1996,  2197,  2048,  3152,  1012,\n",
      "          3422,  1045,  1011,  6398,  2507,  2014,  3319,  1997, 10693,  2015,\n",
      "          6745,  1090,  1012,  2045,  2003,  2166,  3458, 10693,  1010,  2174,\n",
      "          1012,  1996,  2414,  2121,  2038,  6361,  1037,  2694,  3185,  2170,\n",
      "          1000,  2026,  2879,  2990,  1010,  1000,  2055,  3166, 18254,  4232,\n",
      "         11382, 14353,  1998,  2010,  2365,  1010,  2349,  2005,  2713,  2101,\n",
      "          2023,  2095,  1012,  2002,  2097,  2036,  3711,  1999,  1000,  2285,\n",
      "          3337,  1010,  1000,  2019,  2827,  2143,  2055,  2176,  3337,  2040,\n",
      "          4019,  2019, 18504,  1012,  3041,  2023,  2095,  1010,  2002,  2081,\n",
      "          2010,  2754,  2834,  2652,  1037, 12364, 10563,  1999,  2848, 21146,\n",
      "         12494,  2015,  1000,  1041, 28940,  2271,  1012,  1000,  5564,  1010,\n",
      "          2002,  2003, 15515,  2005,  2130,  3553,  2865, 17423,  2085,  2008,\n",
      "          2002,  2015, 10142,  2019,  4639,  1024,  1000,  1045,  2074,  2228,\n",
      "         10047,  2183,  2000,  2022,  2062,  4066,  1997,  4189,  2208,  1010,\n",
      "          1000,  2002,  2409, 26665,  1012,  1041,  1011,  5653,  2000,  1037,\n",
      "          2767,  1012,   102]])\n",
      "543\n"
     ]
    }
   ],
   "source": [
    "filter_text = filter_data(data[0]['article'])\n",
    "\n",
    "print(filter_text)\n",
    "\n",
    "sample_input = np.array(filter_text)\n",
    "tokenizer_output = tokenizer(text = sample_input.tolist(), return_tensors = 'pt')\n",
    "tokens = tokenizer_output['input_ids']\n",
    "\n",
    "print(tokens)\n",
    "print(len(tokens[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "83dabc46",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████| 287113/287113 [00:44<00:00, 6498.04it/s]\n"
     ]
    }
   ],
   "source": [
    "#faster opration can be done using multithread \n",
    "\n",
    "from tqdm import tqdm \n",
    "\n",
    "train_data = []\n",
    "\n",
    "for i in tqdm(range(len(data))):\n",
    "    filter_d = filter_data(data[i]['article'])\n",
    "    train_data.append(filter_d)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "cec8eb31",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "287113"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(train_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "7b91ee1f",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data = train_data[:10000]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "26dbb8b6",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data = CreateDataset(train_data, tokenizer)\n",
    "dataloader = DataLoader(train_data, batch_size = 8, collate_fn = data_collate)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "bf564d99",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1250"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(dataloader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "2b917a91",
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch import nn, Tensor\n",
    "from torch.nn import TransformerEncoder, TransformerEncoderLayer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "1a60311f",
   "metadata": {},
   "outputs": [],
   "source": [
    "class PositionalEncoding(nn.Module):\n",
    "\n",
    "    def __init__(self, d_model, dropout=0.1, max_len=5000):\n",
    "        super(PositionalEncoding, self).__init__()\n",
    "        self.dropout = nn.Dropout(p=dropout)\n",
    "\n",
    "        pe = torch.zeros(max_len, d_model)\n",
    "        position = torch.arange(0, max_len, dtype=torch.float).unsqueeze(1)\n",
    "        div_term = torch.exp(torch.arange(0, d_model, 2).float() * (-math.log(10000.0) / d_model))\n",
    "        pe[:, 0::2] = torch.sin(position * div_term)\n",
    "        pe[:, 1::2] = torch.cos(position * div_term)\n",
    "        pe = pe.unsqueeze(0).transpose(0, 1)\n",
    "        self.register_buffer('pe', pe)\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = x + self.pe[:x.size(0), :]\n",
    "        return self.dropout(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "6b1f8b27",
   "metadata": {},
   "outputs": [],
   "source": [
    "#model \n",
    "class TransformerModel(nn.Module):\n",
    "    \n",
    "    def __init__(self, ntokens, ninp, nhead, nhid, nlayers, dropout = 0.5):\n",
    "        super(TransformerModel, self).__init__()\n",
    "        self.model_type = \"Transformer\"\n",
    "        self.pos_encoder = PositionalEncoding(ninp, dropout)\n",
    "        encoder_layer = TransformerEncoderLayer(ninp, nhead, nhid, dropout, batch_first = True)\n",
    "        self.transformer_encoder = TransformerEncoder(encoder_layer, nlayers)\n",
    "        self.encoder = nn.Embedding(ntokens, ninp)\n",
    "        self.ninp = ninp\n",
    "        self.decoder = nn.Linear(ninp, ntokens)\n",
    "        \n",
    "        self.init_weights()\n",
    "        \n",
    "    def generate_square_subsequent_mask(self, sz):\n",
    "        \n",
    "        '''\n",
    "        We generate the mask to prevent the transformer from seeing future tokens\n",
    "        Square matrix is created with elements below the diagonal = 0\n",
    "        Conver the mask to float, all zeros are replaced with -inf(indicating no access to elements) \n",
    "        and 1 with 0.0(this apporation does not changes the magnitude but influences the output)\n",
    "        '''\n",
    "        mask = (torch.triu(torch.ones(sz, sz)) == 1).transpose(0, 1)\n",
    "        mask = mask.float().masked_fill(mask == 0, float('-inf')).masked_fill(mask == 1, float(0.0))\n",
    "        \n",
    "        return mask\n",
    "    \n",
    "    def init_weights(self):\n",
    "        initrange = 0.1\n",
    "        self.encoder.weight.data.uniform_(-initrange, initrange)\n",
    "        self.decoder.bias.data.zero_()\n",
    "        self.decoder.weight.data.uniform_(-initrange, initrange)\n",
    "        \n",
    "    def forward(self, src, src_mask):\n",
    "        src = self.encoder(src) * math.sqrt(self.ninp)\n",
    "        src = self.pos_encoder(src)\n",
    "        \n",
    "        output = self.transformer_encoder(src, src_mask)\n",
    "        output = self.decoder(output)\n",
    "        return output "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "1b0f3c91",
   "metadata": {},
   "outputs": [],
   "source": [
    "if not torch.backends.mps.is_available():\n",
    "    if not torch.backends.mps.is_built():\n",
    "        print(\"MPS not available because the current PyTorch install was not \"\n",
    "              \"built with MPS enabled.\")\n",
    "    else:\n",
    "        print(\"MPS not available because the current MacOS version is not 12.3+ \"\n",
    "              \"and/or you do not have an MPS-enabled device on this machine.\")\n",
    "\n",
    "else:\n",
    "    mps_device = torch.device(\"mps\")\n",
    "\n",
    "    ntokens = tokenizer.vocab_size \n",
    "    emsize = 512 # embedding dimension\n",
    "\n",
    "    nhid = 100 # the dimension of the feedforward network model in nn.TransformerEncoder\n",
    "\n",
    "    nlayers = 5 # the number of nn.TransformerEncoderLayer in nn.TransformerEncoder\n",
    "\n",
    "    nhead = 4 # the number of heads in the multiheadattention models\n",
    "\n",
    "    dropout = 0.2 # the dropout value\n",
    "\n",
    "    model = TransformerModel(ntokens, emsize, nhead, nhid, nlayers, dropout).to(mps_device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "9534035d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TransformerModel(\n",
       "  (pos_encoder): PositionalEncoding(\n",
       "    (dropout): Dropout(p=0.2, inplace=False)\n",
       "  )\n",
       "  (transformer_encoder): TransformerEncoder(\n",
       "    (layers): ModuleList(\n",
       "      (0-4): 5 x TransformerEncoderLayer(\n",
       "        (self_attn): MultiheadAttention(\n",
       "          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)\n",
       "        )\n",
       "        (linear1): Linear(in_features=512, out_features=100, bias=True)\n",
       "        (dropout): Dropout(p=0.2, inplace=False)\n",
       "        (linear2): Linear(in_features=100, out_features=512, bias=True)\n",
       "        (norm1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "        (norm2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "        (dropout1): Dropout(p=0.2, inplace=False)\n",
       "        (dropout2): Dropout(p=0.2, inplace=False)\n",
       "      )\n",
       "    )\n",
       "  )\n",
       "  (encoder): Embedding(30522, 512)\n",
       "  (decoder): Linear(in_features=512, out_features=30522, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "f2f6e6e8",
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(model, dataloader):\n",
    "    model.train()\n",
    "    epochs = 50\n",
    "    total_loss = 0\n",
    "    criterion = nn.CrossEntropyLoss()\n",
    "    optim = torch.optim.AdamW(model.parameters(), lr = 0.1)\n",
    "    \n",
    "    for epoch in tqdm(range(epochs)):\n",
    "        for batch in tqdm(dataloader):\n",
    "            optim.zero_grad()\n",
    "            input = batch['input_ids'].clone()\n",
    "            \n",
    "            src_mask = model.generate_square_subsequent_mask(batch['input_ids'].size(1))\n",
    "            \n",
    "            rand_value = torch.rand(batch.input_ids.shape)\n",
    "            rand_mask = (rand_value < 0.15) * (input != 101) * (input != 102) * (input != 0)\n",
    "            \n",
    "            mask_idx=(rand_mask.flatten() == True).nonzero().view(-1)\n",
    "            \n",
    "            input = input.flatten()\n",
    "            input[mask_idx] = 103\n",
    "            input = input.view(batch['input_ids'].size())\n",
    "            \n",
    "            out = model(input.to(mps_device), src_mask.to(mps_device))\n",
    "            loss = criterion(out.view(-1, ntokens), batch['input_ids'].view(-1).to(mps_device))\n",
    "            total_loss += loss\n",
    "            loss.backward()\n",
    "            optim.step()\n",
    "            \n",
    "        print(\"Epoch: {} -> loss: {}\".format(epoch+1, total_loss/(len(dataloader)*epoch+1)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dba84f42",
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.mps.empty_cache()\n",
    "train(model, dataloader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9199166d",
   "metadata": {},
   "outputs": [],
   "source": [
    "param_size = 0\n",
    "for param in model.parameters():\n",
    "    param_size += param.nelement() * param.element_size()\n",
    "    \n",
    "buffer_size = 0\n",
    "for buffer in model.buffers():\n",
    "    buffer_size += buffer.nelement() * buffer.element_size()\n",
    "\n",
    "size_all_mb = (param_size + buffer_size) / 1024**2\n",
    "print('model size: {:.3f}MB'.format(size_all_mb))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3262b270",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch \n",
    "\n",
    "one_mat = torch.ones(4, 4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f211bd6e",
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.triu(one_mat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3f89fa4c",
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.triu(one_mat == 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0bcfa751",
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.triu(one_mat == 1).transpose(0, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cf5e6c34",
   "metadata": {},
   "outputs": [],
   "source": [
    "mat = torch.triu(one_mat == 1).transpose(0, 1).float()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e8e11dcb",
   "metadata": {},
   "outputs": [],
   "source": [
    "mat.masked_fill(mat == 0, float('-inf')).masked_fill(mat == 1, float(0.0))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3575567c",
   "metadata": {},
   "source": [
    "## Optimizing GPU usage"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "5517c7be",
   "metadata": {},
   "outputs": [],
   "source": [
    "from accelerate import Accelerator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "69718feb",
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import TrainingArguments\n",
    "\n",
    "training_args = TrainingArguments(\n",
    "    per_device_train_batch_size=1,\n",
    "    learning_rate = 0.1,\n",
    "    gradient_accumulation_steps=4,\n",
    "    gradient_checkpointing=True,\n",
    "    #fp16=True, # can only be done with CUDA \n",
    "    output_dir = \"./model_output\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "54ee9df7",
   "metadata": {},
   "outputs": [
    {
     "ename": "AttributeError",
     "evalue": "'TransformerModel' object has no attribute 'gradient_checkpointing_enable'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[26], line 4\u001b[0m\n\u001b[1;32m      1\u001b[0m dataloader \u001b[38;5;241m=\u001b[39m DataLoader(train_data, batch_size\u001b[38;5;241m=\u001b[39mtraining_args\u001b[38;5;241m.\u001b[39mper_device_train_batch_size)\n\u001b[1;32m      3\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m training_args\u001b[38;5;241m.\u001b[39mgradient_checkpointing:\n\u001b[0;32m----> 4\u001b[0m     \u001b[43mmodel\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mgradient_checkpointing_enable\u001b[49m()\n\u001b[1;32m      6\u001b[0m accelerator \u001b[38;5;241m=\u001b[39m Accelerator()\n\u001b[1;32m      7\u001b[0m model, optimizer, dataloader \u001b[38;5;241m=\u001b[39m accelerator\u001b[38;5;241m.\u001b[39mprepare(model, adam_bnb_optim, dataloader)\n",
      "File \u001b[0;32m~/opt/miniconda3/envs/m1_gpu/lib/python3.8/site-packages/torch/nn/modules/module.py:1695\u001b[0m, in \u001b[0;36mModule.__getattr__\u001b[0;34m(self, name)\u001b[0m\n\u001b[1;32m   1693\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m name \u001b[38;5;129;01min\u001b[39;00m modules:\n\u001b[1;32m   1694\u001b[0m         \u001b[38;5;28;01mreturn\u001b[39;00m modules[name]\n\u001b[0;32m-> 1695\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mAttributeError\u001b[39;00m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mtype\u001b[39m(\u001b[38;5;28mself\u001b[39m)\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__name__\u001b[39m\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m object has no attribute \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mname\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n",
      "\u001b[0;31mAttributeError\u001b[0m: 'TransformerModel' object has no attribute 'gradient_checkpointing_enable'"
     ]
    }
   ],
   "source": [
    "dataloader = DataLoader(train_data, batch_size=training_args.per_device_train_batch_size)\n",
    "\n",
    "if training_args.gradient_checkpointing:\n",
    "    model.gradient_checkpointing_enable()\n",
    "\n",
    "accelerator = Accelerator()\n",
    "model, optimizer, dataloader = accelerator.prepare(model, adam_bnb_optim, dataloader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "dc8bc60c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['transformer_encoder.layers.0.self_attn.out_proj.weight', 'transformer_encoder.layers.0.self_attn.out_proj.bias', 'transformer_encoder.layers.0.self_attn.in_proj_weight', 'transformer_encoder.layers.0.self_attn.q_proj_weight', 'transformer_encoder.layers.0.self_attn.k_proj_weight', 'transformer_encoder.layers.0.self_attn.v_proj_weight', 'transformer_encoder.layers.0.self_attn.in_proj_bias', 'transformer_encoder.layers.0.linear1.weight', 'transformer_encoder.layers.0.linear1.bias', 'transformer_encoder.layers.0.linear2.weight', 'transformer_encoder.layers.0.linear2.bias', 'transformer_encoder.layers.1.self_attn.out_proj.weight', 'transformer_encoder.layers.1.self_attn.out_proj.bias', 'transformer_encoder.layers.1.self_attn.in_proj_weight', 'transformer_encoder.layers.1.self_attn.q_proj_weight', 'transformer_encoder.layers.1.self_attn.k_proj_weight', 'transformer_encoder.layers.1.self_attn.v_proj_weight', 'transformer_encoder.layers.1.self_attn.in_proj_bias', 'transformer_encoder.layers.1.linear1.weight', 'transformer_encoder.layers.1.linear1.bias', 'transformer_encoder.layers.1.linear2.weight', 'transformer_encoder.layers.1.linear2.bias', 'transformer_encoder.layers.2.self_attn.out_proj.weight', 'transformer_encoder.layers.2.self_attn.out_proj.bias', 'transformer_encoder.layers.2.self_attn.in_proj_weight', 'transformer_encoder.layers.2.self_attn.q_proj_weight', 'transformer_encoder.layers.2.self_attn.k_proj_weight', 'transformer_encoder.layers.2.self_attn.v_proj_weight', 'transformer_encoder.layers.2.self_attn.in_proj_bias', 'transformer_encoder.layers.2.linear1.weight', 'transformer_encoder.layers.2.linear1.bias', 'transformer_encoder.layers.2.linear2.weight', 'transformer_encoder.layers.2.linear2.bias', 'transformer_encoder.layers.3.self_attn.out_proj.weight', 'transformer_encoder.layers.3.self_attn.out_proj.bias', 'transformer_encoder.layers.3.self_attn.in_proj_weight', 'transformer_encoder.layers.3.self_attn.q_proj_weight', 'transformer_encoder.layers.3.self_attn.k_proj_weight', 'transformer_encoder.layers.3.self_attn.v_proj_weight', 'transformer_encoder.layers.3.self_attn.in_proj_bias', 'transformer_encoder.layers.3.linear1.weight', 'transformer_encoder.layers.3.linear1.bias', 'transformer_encoder.layers.3.linear2.weight', 'transformer_encoder.layers.3.linear2.bias', 'transformer_encoder.layers.4.self_attn.out_proj.weight', 'transformer_encoder.layers.4.self_attn.out_proj.bias', 'transformer_encoder.layers.4.self_attn.in_proj_weight', 'transformer_encoder.layers.4.self_attn.q_proj_weight', 'transformer_encoder.layers.4.self_attn.k_proj_weight', 'transformer_encoder.layers.4.self_attn.v_proj_weight', 'transformer_encoder.layers.4.self_attn.in_proj_bias', 'transformer_encoder.layers.4.linear1.weight', 'transformer_encoder.layers.4.linear1.bias', 'transformer_encoder.layers.4.linear2.weight', 'transformer_encoder.layers.4.linear2.bias', 'encoder.weight', 'decoder.weight', 'decoder.bias']\n"
     ]
    }
   ],
   "source": [
    "from transformers.trainer_pt_utils import get_parameter_names\n",
    "\n",
    "decay_parameters = get_parameter_names(model, [nn.LayerNorm])\n",
    "print(decay_parameters)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "5fa50f2c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['transformer_encoder.layers.0.self_attn.out_proj.weight', 'transformer_encoder.layers.0.self_attn.in_proj_weight', 'transformer_encoder.layers.0.self_attn.q_proj_weight', 'transformer_encoder.layers.0.self_attn.k_proj_weight', 'transformer_encoder.layers.0.self_attn.v_proj_weight', 'transformer_encoder.layers.0.linear1.weight', 'transformer_encoder.layers.0.linear2.weight', 'transformer_encoder.layers.1.self_attn.out_proj.weight', 'transformer_encoder.layers.1.self_attn.in_proj_weight', 'transformer_encoder.layers.1.self_attn.q_proj_weight', 'transformer_encoder.layers.1.self_attn.k_proj_weight', 'transformer_encoder.layers.1.self_attn.v_proj_weight', 'transformer_encoder.layers.1.linear1.weight', 'transformer_encoder.layers.1.linear2.weight', 'transformer_encoder.layers.2.self_attn.out_proj.weight', 'transformer_encoder.layers.2.self_attn.in_proj_weight', 'transformer_encoder.layers.2.self_attn.q_proj_weight', 'transformer_encoder.layers.2.self_attn.k_proj_weight', 'transformer_encoder.layers.2.self_attn.v_proj_weight', 'transformer_encoder.layers.2.linear1.weight', 'transformer_encoder.layers.2.linear2.weight', 'transformer_encoder.layers.3.self_attn.out_proj.weight', 'transformer_encoder.layers.3.self_attn.in_proj_weight', 'transformer_encoder.layers.3.self_attn.q_proj_weight', 'transformer_encoder.layers.3.self_attn.k_proj_weight', 'transformer_encoder.layers.3.self_attn.v_proj_weight', 'transformer_encoder.layers.3.linear1.weight', 'transformer_encoder.layers.3.linear2.weight', 'transformer_encoder.layers.4.self_attn.out_proj.weight', 'transformer_encoder.layers.4.self_attn.in_proj_weight', 'transformer_encoder.layers.4.self_attn.q_proj_weight', 'transformer_encoder.layers.4.self_attn.k_proj_weight', 'transformer_encoder.layers.4.self_attn.v_proj_weight', 'transformer_encoder.layers.4.linear1.weight', 'transformer_encoder.layers.4.linear2.weight', 'encoder.weight', 'decoder.weight']\n"
     ]
    }
   ],
   "source": [
    "decay_parameters = [name for name in decay_parameters if \"bias\" not in name]\n",
    "print(decay_parameters)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "8867af66",
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer_grouped_parameters = [\n",
    "    {\n",
    "        \"params\": [p for n, p in model.named_parameters() if n in decay_parameters],\n",
    "        \"weight_decay\": training_args.weight_decay,\n",
    "    },\n",
    "    {\n",
    "        \"params\": [p for n, p in model.named_parameters() if n not in decay_parameters],\n",
    "        \"weight_decay\": 0.0,\n",
    "    },\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "b7213a72",
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer_kwargs = {\n",
    "    \"betas\": (training_args.adam_beta1, training_args.adam_beta2),\n",
    "    \"eps\": training_args.adam_epsilon,\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "6f0746b3",
   "metadata": {},
   "outputs": [
    {
     "ename": "ModuleNotFoundError",
     "evalue": "No module named 'bitsandbytes'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mModuleNotFoundError\u001b[0m                       Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[34], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mbitsandbytes\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m \u001b[38;5;21;01mbnb\u001b[39;00m\n\u001b[1;32m      3\u001b[0m optimizer_kwargs[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mlr\u001b[39m\u001b[38;5;124m\"\u001b[39m] \u001b[38;5;241m=\u001b[39m training_args\u001b[38;5;241m.\u001b[39mlearning_rate\n\u001b[1;32m      5\u001b[0m adam_bnb_optim \u001b[38;5;241m=\u001b[39m bnb\u001b[38;5;241m.\u001b[39moptim\u001b[38;5;241m.\u001b[39mAdam8bit(\n\u001b[1;32m      6\u001b[0m     optimizer_grouped_parameters,\n\u001b[1;32m      7\u001b[0m     betas\u001b[38;5;241m=\u001b[39m(training_args\u001b[38;5;241m.\u001b[39madam_beta1, training_args\u001b[38;5;241m.\u001b[39madam_beta2),\n\u001b[1;32m      8\u001b[0m     eps\u001b[38;5;241m=\u001b[39mtraining_args\u001b[38;5;241m.\u001b[39madam_epsilon,\n\u001b[1;32m      9\u001b[0m     lr\u001b[38;5;241m=\u001b[39mtraining_args\u001b[38;5;241m.\u001b[39mlearning_rate,\n\u001b[1;32m     10\u001b[0m )\n",
      "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'bitsandbytes'"
     ]
    }
   ],
   "source": [
    "import bitsandbytes as bnb\n",
    "\n",
    "optimizer_kwargs[\"lr\"] = training_args.learning_rate\n",
    "\n",
    "adam_bnb_optim = bnb.optim.Adam8bit(\n",
    "    optimizer_grouped_parameters,\n",
    "    betas=(training_args.adam_beta1, training_args.adam_beta2),\n",
    "    eps=training_args.adam_epsilon,\n",
    "    lr=training_args.learning_rate,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e4b7c4f1",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.17"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}