dataset_stat.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from flearn.utils.model_utils import read_data\n",
    "from collections import Counter\n",
    "import os\n",
    "\n",
    "options = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def count_label(dataset):\n",
    "    cont = Counter()\n",
    "    for data in dataset.values():\n",
    "        cont += Counter(data['y'])\n",
    "    return cont"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "772 Counter({'4': 20058, '0': 12241}) 32299\n",
      "772 Counter({'4': 5271, '0': 3213}) 8484\n"
     ]
    }
   ],
   "source": [
    "\"\"\" MNIST \"\"\"\n",
    "\n",
    "options['dataset'] = 'sent140'\n",
    "train_path = os.path.join('data', options['dataset'], 'data', 'train')\n",
    "test_path = os.path.join('data', options['dataset'], 'data', 'test')\n",
    "dataset = read_data(train_path, test_path) # return clientID, group, train_data, test_data\n",
    "\n",
    "\n",
    "test_cont = count_label(dataset[3])\n",
    "train_cont = count_label(dataset[2])\n",
    "print(len(dataset[0]), train_cont, sum(train_cont.values()))\n",
    "print(len(dataset[0]), test_cont, sum(test_cont.values()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[8.0, 8.0, 8.0, 7.0, 6.0, 9.0, 6.0, 8.0, 9.0, 6.0, 6.0, 0.0, 9.0, 8.0, 7.0, 6.0, 6.0, 8.0, 9.0, 8.0, 8.0, 8.0, 6.0, 0.0, 8.0, 9.0, 8.0, 8.0, 7.0, 0.0, 8.0, 8.0, 6.0, 6.0, 0.0, 8.0, 6.0, 6.0, 0.0, 6.0, 8.0, 7.0, 8.0]\n"
     ]
    }
   ],
   "source": [
    "train_data = dataset[2]\n",
    "test_data = dataset[3]\n",
    "\n",
    "train_size = [len(data['y']) for data in train_data.values()]\n",
    "#print(train_size)\n",
    "\n",
    "train_client_list = list(train_data.keys())\n",
    "print(train_data[train_client_list[0]]['y'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "200 Counter({4.0: 2250, 2.0: 2175, 3.0: 2045, 5.0: 1556, 0.0: 1450, 7.0: 1442, 6.0: 1427, 1.0: 1399, 8.0: 1353, 9.0: 1324})\n",
      "200 Counter({4.0: 254, 2.0: 246, 3.0: 233, 5.0: 201, 0.0: 182, 6.0: 171, 7.0: 166, 9.0: 165, 1.0: 156, 8.0: 150})\n"
     ]
    }
   ],
   "source": [
    "\"\"\" NIST \"\"\"\n",
    "options['dataset'] = 'nist'\n",
    "train_path = os.path.join('data', options['dataset'], 'data', 'train')\n",
    "test_path = os.path.join('data', options['dataset'], 'data', 'test')\n",
    "dataset = read_data(train_path, test_path) # return clientID, group, train_data, test_data\n",
    "\n",
    "test_cont = count_label(dataset[3])\n",
    "train_cont = count_label(dataset[2])\n",
    "print(len(dataset[0]), train_cont)\n",
    "print(len(dataset[0]), test_cont)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "772 Counter({'4': 20058, '0': 12241})\n",
      "772 Counter({'4': 5271, '0': 3213})\n"
     ]
    }
   ],
   "source": [
    "\"\"\" Sent140 \"\"\"\n",
    "options['dataset'] = 'sent140'\n",
    "train_path = os.path.join('data', options['dataset'], 'data', 'train')\n",
    "test_path = os.path.join('data', options['dataset'], 'data', 'test')\n",
    "dataset = read_data(train_path, test_path)\n",
    "\n",
    "test_cont = count_label(dataset[3])\n",
    "train_cont = count_label(dataset[2])\n",
    "print(len(dataset[0]), train_cont)\n",
    "print(len(dataset[0]), test_cont)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "143 Counter({' ': 77055, 'e': 36543, 't': 26556, 'o': 25728, 'a': 21852, 'h': 19959, 's': 19663, 'n': 19570, 'r': 18910, 'i': 18326, 'l': 13187, 'd': 12357, 'u': 10412, 'm': 8407, 'y': 7716, ',': 7334, 'w': 6755, 'f': 6370, 'c': 6240, 'g': 5333, 'p': 4353, 'b': 4223, '.': 3959, 'v': 3052, 'k': 2802, \"'\": 2782, 'I': 2658, 'T': 2051, 'A': 1960, ';': 1625, 'W': 1143, 'E': 962, '?': 958, 'S': 941, 'O': 891, '!': 878, 'H': 871, 'B': 738, '-': 714, 'M': 678, 'N': 668, 'F': 608, 'C': 602, 'L': 600, 'G': 534, 'R': 528, 'D': 448, 'P': 422, 'x': 416, 'Y': 395, 'j': 287, 'K': 251, 'U': 239, 'q': 207, ':': 184, ']': 170, '[': 161, 'J': 122, 'z': 99, 'V': 90, 'Q': 48, '\"': 14, 'X': 9, 'Z': 8, '&': 5, '(': 1, ')': 1})\n",
      "143 Counter({' ': 19280, 'e': 9209, 't': 6460, 'o': 6357, 'a': 5643, 'h': 4932, 'n': 4914, 's': 4890, 'r': 4743, 'i': 4510, 'l': 3365, 'd': 3144, 'u': 2521, 'm': 2168, 'y': 1998, ',': 1848, 'w': 1688, 'f': 1575, 'c': 1567, 'g': 1363, 'p': 1101, 'b': 1077, '.': 948, 'v': 789, 'I': 700, \"'\": 694, 'k': 681, 'T': 498, 'A': 438, ';': 417, 'W': 321, 'E': 259, 'S': 229, 'H': 217, '?': 214, '!': 212, 'O': 189, 'M': 183, '-': 169, 'C': 164, 'L': 162, 'N': 156, 'B': 152, 'F': 141, 'R': 141, 'D': 118, 'G': 114, 'P': 112, 'x': 106, 'Y': 99, 'j': 81, 'U': 69, 'K': 66, ':': 56, 'q': 47, '[': 41, 'J': 34, 'z': 33, ']': 31, 'V': 22, 'Q': 14, 'X': 3, '\"': 2, '&': 1, '5': 1})\n"
     ]
    }
   ],
   "source": [
    "\"\"\" shakespeare \"\"\"\n",
    "options['dataset'] = 'shakespeare'\n",
    "train_path = os.path.join('data', options['dataset'], 'data', 'train')\n",
    "test_path = os.path.join('data', options['dataset'], 'data', 'test')\n",
    "dataset = read_data(train_path, test_path)\n",
    "\n",
    "test_cont = count_label(dataset[3])\n",
    "train_cont = count_label(dataset[2])\n",
    "print(len(dataset[0]), train_cont)\n",
    "print(len(dataset[0]), test_cont)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}