examples/tutorial/BERTVerifierMRC.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "political-table",
   "metadata": {},
   "source": [
    "# BERTVerifierMRC\n",
    "\n",
    "可用的中文预训练参数：[`bert-base`](https://storage.googleapis.com/bert_models/2018_11_03/chinese_L-12_H-768_A-12.zip),[`roberta-wwm-ext-base`](https://drive.google.com/uc?export=download&id=1jMAKIJmPn7kADgD3yQZhpsqM-IRM1qZt),[`roberta-wwm-ext-large`](https://drive.google.com/uc?export=download&id=1dtad0FFzG11CBsawu8hvwwzU2R0FDI94),[`macbert-base`](https://drive.google.com/uc?export=download&id=1aV69OhYzIwj_hn-kO1RiBa-m8QAusQ5b),[`macbert-large`](https://drive.google.com/uc?export=download&id=1lWYxnk1EqTA2Q20_IShxBrCPc5VSDCkT)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "rotary-academy",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "v2.5.2\n"
     ]
    }
   ],
   "source": [
    "import uf\n",
    "\n",
    "print(uf.__version__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "velvet-symbol",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "uf.BERTVerifierMRC(\n",
      "    config_file=\"../../ref/bert_config.json\",\n",
      "    vocab_file=\"../../ref/vocab.txt\",\n",
      "    max_seq_length=256,\n",
      "    init_checkpoint=None,\n",
      "    output_dir=None,\n",
      "    gpu_ids=None,\n",
      "    do_lower_case=True,\n",
      "    drop_pooler=False,\n",
      "    truncate_method=\"longer-FO\",\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "model = uf.BERTVerifierMRC(\"../../ref/bert_config.json\", \"../../ref/vocab.txt\")\n",
    "print(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "drawn-breathing",
   "metadata": {},
   "outputs": [],
   "source": [
    "X = [{\"doc\": \"天亮以前说再见，笑着泪流满面。去迎接应该你的，更好的明天\", \"ques\": \"何时说的再见\"}, \n",
    "     {\"doc\": \"他想知道那是谁，为何总沉默寡言。人群中也算抢眼，抢眼的孤独难免\", \"ques\": \"抢眼的如何\"}]\n",
    "y = [{\"text\": \"天亮以前\", \"answer_start\": 0}, {\"text\": \"孤独难免\", \"answer_start\": 27}]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "vanilla-fault",
   "metadata": {},
   "source": [
    "# 训练"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "excellent-religion",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/geyingli/Library/Python/3.8/lib/python/site-packages/uf-2.5.2-py3.8.egg/uf/apps/bert/bert.py:274: UserWarning: `tf.layers.dense` is deprecated and will be removed in a future version. Please use `tf.keras.layers.Dense` instead.\n",
      "  query_layer = tf.layers.dense(\n",
      "/Users/geyingli/Library/Python/3.8/lib/python/site-packages/uf-2.5.2-py3.8.egg/uf/apps/bert/bert.py:284: UserWarning: `tf.layers.dense` is deprecated and will be removed in a future version. Please use `tf.keras.layers.Dense` instead.\n",
      "  key_layer = tf.layers.dense(\n",
      "/Users/geyingli/Library/Python/3.8/lib/python/site-packages/uf-2.5.2-py3.8.egg/uf/apps/bert/bert.py:294: UserWarning: `tf.layers.dense` is deprecated and will be removed in a future version. Please use `tf.keras.layers.Dense` instead.\n",
      "  value_layer = tf.layers.dense(\n",
      "/Users/geyingli/Library/Python/3.8/lib/python/site-packages/uf-2.5.2-py3.8.egg/uf/apps/bert/bert.py:418: UserWarning: `tf.layers.dense` is deprecated and will be removed in a future version. Please use `tf.keras.layers.Dense` instead.\n",
      "  attention_output = tf.layers.dense(\n",
      "/Users/geyingli/Library/Python/3.8/lib/python/site-packages/uf-2.5.2-py3.8.egg/uf/apps/bert/bert.py:430: UserWarning: `tf.layers.dense` is deprecated and will be removed in a future version. Please use `tf.keras.layers.Dense` instead.\n",
      "  intermediate_output = tf.layers.dense(\n",
      "/Users/geyingli/Library/Python/3.8/lib/python/site-packages/uf-2.5.2-py3.8.egg/uf/apps/bert/bert.py:440: UserWarning: `tf.layers.dense` is deprecated and will be removed in a future version. Please use `tf.keras.layers.Dense` instead.\n",
      "  layer_output = tf.layers.dense(\n",
      "/Users/geyingli/Library/Python/3.8/lib/python/site-packages/uf-2.5.2-py3.8.egg/uf/apps/bert/bert.py:109: UserWarning: `tf.layers.dense` is deprecated and will be removed in a future version. Please use `tf.keras.layers.Dense` instead.\n",
      "  self.pooled_output = tf.layers.dense(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Build graph with 306,812,172 parameters (among which 102,270,724 are trainable)\n",
      "INFO:tensorflow:Running local_init_op\n",
      "INFO:tensorflow:Done running local_init_op\n",
      "INFO:tensorflow:Running training on 2 samples (step 0 -> 20)\n",
      "INFO:tensorflow:step 1, has_ans_accuracy 1.0000, exact_match 0.0000, f1 0.0417, verifier_loss 0.683821, mrc_loss 5.472636, 0.11 steps/sec, 0.22 examples/sec\n",
      "INFO:tensorflow:step 2, has_ans_accuracy 0.0000, exact_match 0.0000, f1 0.0000, verifier_loss 0.747386, mrc_loss 5.389949, 0.40 steps/sec, 0.79 examples/sec\n",
      "INFO:tensorflow:step 3, has_ans_accuracy 1.0000, exact_match 0.0000, f1 0.0000, verifier_loss 0.243052, mrc_loss 3.845150, 0.40 steps/sec, 0.81 examples/sec\n",
      "INFO:tensorflow:step 4, has_ans_accuracy 1.0000, exact_match 1.0000, f1 1.0000, verifier_loss 0.040429, mrc_loss 1.648562, 0.39 steps/sec, 0.78 examples/sec\n",
      "INFO:tensorflow:step 5, has_ans_accuracy 1.0000, exact_match 1.0000, f1 1.0000, verifier_loss 0.016606, mrc_loss 0.719559, 0.40 steps/sec, 0.79 examples/sec\n",
      "INFO:tensorflow:step 6, has_ans_accuracy 1.0000, exact_match 1.0000, f1 1.0000, verifier_loss 0.010961, mrc_loss 0.180172, 0.40 steps/sec, 0.80 examples/sec\n",
      "INFO:tensorflow:step 7, has_ans_accuracy 1.0000, exact_match 1.0000, f1 1.0000, verifier_loss 0.013351, mrc_loss 0.031222, 0.39 steps/sec, 0.79 examples/sec\n",
      "INFO:tensorflow:step 8, has_ans_accuracy 1.0000, exact_match 1.0000, f1 1.0000, verifier_loss 0.010443, mrc_loss 0.005329, 0.38 steps/sec, 0.76 examples/sec\n",
      "INFO:tensorflow:step 9, has_ans_accuracy 1.0000, exact_match 1.0000, f1 1.0000, verifier_loss 0.006497, mrc_loss 0.001460, 0.33 steps/sec, 0.66 examples/sec\n",
      "INFO:tensorflow:step 10, has_ans_accuracy 1.0000, exact_match 1.0000, f1 1.0000, verifier_loss 0.004072, mrc_loss 0.000775, 0.24 steps/sec, 0.49 examples/sec\n",
      "INFO:tensorflow:step 11, has_ans_accuracy 1.0000, exact_match 1.0000, f1 1.0000, verifier_loss 0.002313, mrc_loss 0.000273, 0.23 steps/sec, 0.46 examples/sec\n",
      "INFO:tensorflow:step 12, has_ans_accuracy 1.0000, exact_match 1.0000, f1 1.0000, verifier_loss 0.001782, mrc_loss 0.000474, 0.20 steps/sec, 0.40 examples/sec\n",
      "INFO:tensorflow:step 13, has_ans_accuracy 1.0000, exact_match 1.0000, f1 1.0000, verifier_loss 0.001248, mrc_loss 0.000374, 0.23 steps/sec, 0.46 examples/sec\n",
      "INFO:tensorflow:step 14, has_ans_accuracy 1.0000, exact_match 1.0000, f1 1.0000, verifier_loss 0.001001, mrc_loss 0.000287, 0.23 steps/sec, 0.46 examples/sec\n",
      "INFO:tensorflow:step 15, has_ans_accuracy 1.0000, exact_match 1.0000, f1 1.0000, verifier_loss 0.000878, mrc_loss 0.000215, 0.24 steps/sec, 0.48 examples/sec\n",
      "INFO:tensorflow:step 16, has_ans_accuracy 1.0000, exact_match 1.0000, f1 1.0000, verifier_loss 0.000789, mrc_loss 0.000179, 0.23 steps/sec, 0.46 examples/sec\n",
      "INFO:tensorflow:step 17, has_ans_accuracy 1.0000, exact_match 1.0000, f1 1.0000, verifier_loss 0.000666, mrc_loss 0.000175, 0.17 steps/sec, 0.34 examples/sec\n",
      "INFO:tensorflow:step 18, has_ans_accuracy 1.0000, exact_match 1.0000, f1 1.0000, verifier_loss 0.000633, mrc_loss 0.000123, 0.19 steps/sec, 0.38 examples/sec\n",
      "INFO:tensorflow:step 19, has_ans_accuracy 1.0000, exact_match 1.0000, f1 1.0000, verifier_loss 0.000637, mrc_loss 0.000201, 0.20 steps/sec, 0.41 examples/sec\n",
      "INFO:tensorflow:step 20, has_ans_accuracy 1.0000, exact_match 1.0000, f1 1.0000, verifier_loss 0.000508, mrc_loss 0.000198, 0.23 steps/sec, 0.46 examples/sec\n"
     ]
    }
   ],
   "source": [
    "model.fit(X, y, total_steps=20)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fallen-hundred",
   "metadata": {},
   "source": [
    "# 推理"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "opposite-pantyhose",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Running inference on 2 samples\n",
      "INFO:tensorflow:process 100.0%, 0.48 examples/sec\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'verifier_probs': array([0.99954945, 0.9995441 ], dtype=float32),\n",
       " 'verifier_preds': array([1, 1]),\n",
       " 'mrc_probs': array([[[1.0310555e-05, 5.8907239e-07, 6.8631931e-07, ...,\n",
       "          1.7751003e-07, 1.9900740e-07, 2.2269458e-07],\n",
       "         [1.6025810e-05, 1.3330276e-06, 7.9587022e-07, ...,\n",
       "          2.9466446e-07, 2.5697855e-07, 2.1447028e-07]],\n",
       " \n",
       "        [[2.7746321e-06, 2.2792210e-07, 1.2146285e-07, ...,\n",
       "          5.1216077e-08, 5.7462870e-08, 6.4900583e-08],\n",
       "         [8.0078889e-06, 6.8668396e-07, 4.3365151e-07, ...,\n",
       "          1.5440277e-07, 1.3572938e-07, 1.1323474e-07]]], dtype=float32),\n",
       " 'mrc_preds': ['天亮以前', '孤独难免']}"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.predict(X)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fundamental-poverty",
   "metadata": {},
   "source": [
    "# 评分"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "accessory-lesbian",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Running scoring on 2 samples\n",
      "INFO:tensorflow:process 100.0%, 0.59 examples/sec\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'has_ans_accuracy': 1.0,\n",
       " 'exact_match': 1.0,\n",
       " 'f1': 1.0,\n",
       " 'verifier_loss': 0.00045330977,\n",
       " 'mrc_loss': 6.8721216e-05}"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.score(X, y)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.9.13 64-bit",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.2"
  },
  "vscode": {
   "interpreter": {
    "hash": "265fd6f62f200408acbbeae0248f34bed9f93569a643842b7a25d2cd76cae5e5"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}