appa.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "plt.rc('figure', figsize=(10, 6))\n",
    "PREVIOUS_MAX_ROWS = pd.options.display.max_rows\n",
    "pd.options.display.max_columns = 20\n",
    "pd.options.display.max_rows = 20\n",
    "pd.options.display.max_colwidth = 80\n",
    "np.set_printoptions(precision=4, suppress=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "rng = np.random.default_rng(seed=12345)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.ones((10, 5)).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.ones((3, 4, 5), dtype=np.float64).strides"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "ints = np.ones(10, dtype=np.uint16)\n",
    "floats = np.ones(10, dtype=np.float32)\n",
    "np.issubdtype(ints.dtype, np.integer)\n",
    "np.issubdtype(floats.dtype, np.floating)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.float64.mro()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.issubdtype(ints.dtype, np.number)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "arr = np.arange(8)\n",
    "arr\n",
    "arr.reshape((4, 2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "arr.reshape((4, 2)).reshape((2, 4))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "arr = np.arange(15)\n",
    "arr.reshape((5, -1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "other_arr = np.ones((3, 5))\n",
    "other_arr.shape\n",
    "arr.reshape(other_arr.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "arr = np.arange(15).reshape((5, 3))\n",
    "arr\n",
    "arr.ravel()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "arr.flatten()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "arr = np.arange(12).reshape((3, 4))\n",
    "arr\n",
    "arr.ravel()\n",
    "arr.ravel('F')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "arr1 = np.array([[1, 2, 3], [4, 5, 6]])\n",
    "arr2 = np.array([[7, 8, 9], [10, 11, 12]])\n",
    "np.concatenate([arr1, arr2], axis=0)\n",
    "np.concatenate([arr1, arr2], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.vstack((arr1, arr2))\n",
    "np.hstack((arr1, arr2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "arr = rng.standard_normal((5, 2))\n",
    "arr\n",
    "first, second, third = np.split(arr, [1, 3])\n",
    "first\n",
    "second\n",
    "third"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "arr = np.arange(6)\n",
    "arr1 = arr.reshape((3, 2))\n",
    "arr2 = rng.standard_normal((3, 2))\n",
    "np.r_[arr1, arr2]\n",
    "np.c_[np.r_[arr1, arr2], arr]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.c_[1:6, -10:-5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "arr = np.arange(3)\n",
    "arr\n",
    "arr.repeat(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "arr.repeat([2, 3, 4])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "arr = rng.standard_normal((2, 2))\n",
    "arr\n",
    "arr.repeat(2, axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "arr.repeat([2, 3], axis=0)\n",
    "arr.repeat([2, 3], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "arr\n",
    "np.tile(arr, 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "arr\n",
    "np.tile(arr, (2, 1))\n",
    "np.tile(arr, (3, 2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "arr = np.arange(10) * 100\n",
    "inds = [7, 1, 2, 6]\n",
    "arr[inds]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "arr.take(inds)\n",
    "arr.put(inds, 42)\n",
    "arr\n",
    "arr.put(inds, [40, 41, 42, 43])\n",
    "arr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "inds = [2, 0, 2, 1]\n",
    "arr = rng.standard_normal((2, 4))\n",
    "arr\n",
    "arr.take(inds, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "arr = np.arange(5)\n",
    "arr\n",
    "arr * 4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "arr = rng.standard_normal((4, 3))\n",
    "arr.mean(0)\n",
    "demeaned = arr - arr.mean(0)\n",
    "demeaned\n",
    "demeaned.mean(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "arr\n",
    "row_means = arr.mean(1)\n",
    "row_means.shape\n",
    "row_means.reshape((4, 1))\n",
    "demeaned = arr - row_means.reshape((4, 1))\n",
    "demeaned.mean(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "arr - arr.mean(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "arr - arr.mean(1).reshape((4, 1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "arr = np.zeros((4, 4))\n",
    "arr_3d = arr[:, np.newaxis, :]\n",
    "arr_3d.shape\n",
    "arr_1d = rng.standard_normal(3)\n",
    "arr_1d[:, np.newaxis]\n",
    "arr_1d[np.newaxis, :]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "arr = rng.standard_normal((3, 4, 5))\n",
    "depth_means = arr.mean(2)\n",
    "depth_means\n",
    "depth_means.shape\n",
    "demeaned = arr - depth_means[:, :, np.newaxis]\n",
    "demeaned.mean(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "arr = np.zeros((4, 3))\n",
    "arr[:] = 5\n",
    "arr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "col = np.array([1.28, -0.42, 0.44, 1.6])\n",
    "arr[:] = col[:, np.newaxis]\n",
    "arr\n",
    "arr[:2] = [[-1.37], [0.509]]\n",
    "arr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "arr = np.arange(10)\n",
    "np.add.reduce(arr)\n",
    "arr.sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "my_rng = np.random.default_rng(12346)  # for reproducibility\n",
    "arr = my_rng.standard_normal((5, 5))\n",
    "arr\n",
    "arr[::2].sort(1) # sort a few rows\n",
    "arr[:, :-1] < arr[:, 1:]\n",
    "np.logical_and.reduce(arr[:, :-1] < arr[:, 1:], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "arr = np.arange(15).reshape((3, 5))\n",
    "np.add.accumulate(arr, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "arr = np.arange(3).repeat([1, 2, 2])\n",
    "arr\n",
    "np.multiply.outer(arr, np.arange(5))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "x, y = rng.standard_normal((3, 4)), rng.standard_normal(5)\n",
    "result = np.subtract.outer(x, y)\n",
    "result.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "arr = np.arange(10)\n",
    "np.add.reduceat(arr, [0, 5, 8])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "arr = np.multiply.outer(np.arange(4), np.arange(5))\n",
    "arr\n",
    "np.add.reduceat(arr, [0, 2, 4], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "def add_elements(x, y):\n",
    "    return x + y\n",
    "add_them = np.frompyfunc(add_elements, 2, 1)\n",
    "add_them(np.arange(8), np.arange(8))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "add_them = np.vectorize(add_elements, otypes=[np.float64])\n",
    "add_them(np.arange(8), np.arange(8))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "arr = rng.standard_normal(10000)\n",
    "%timeit add_them(arr, arr)\n",
    "%timeit np.add(arr, arr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "dtype = [('x', np.float64), ('y', np.int32)]\n",
    "sarr = np.array([(1.5, 6), (np.pi, -2)], dtype=dtype)\n",
    "sarr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "sarr[0]\n",
    "sarr[0]['y']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "sarr['x']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "dtype = [('x', np.int64, 3), ('y', np.int32)]\n",
    "arr = np.zeros(4, dtype=dtype)\n",
    "arr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "arr[0]['x']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "arr['x']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "dtype = [('x', [('a', 'f8'), ('b', 'f4')]), ('y', np.int32)]\n",
    "data = np.array([((1, 2), 5), ((3, 4), 6)], dtype=dtype)\n",
    "data['x']\n",
    "data['y']\n",
    "data['x']['a']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "arr = rng.standard_normal(6)\n",
    "arr.sort()\n",
    "arr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "arr = rng.standard_normal((3, 5))\n",
    "arr\n",
    "arr[:, 0].sort()  # Sort first column values in place\n",
    "arr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "arr = rng.standard_normal(5)\n",
    "arr\n",
    "np.sort(arr)\n",
    "arr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "arr = rng.standard_normal((3, 5))\n",
    "arr\n",
    "arr.sort(axis=1)\n",
    "arr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "arr[:, ::-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "values = np.array([5, 0, 1, 3, 2])\n",
    "indexer = values.argsort()\n",
    "indexer\n",
    "values[indexer]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "arr = rng.standard_normal((3, 5))\n",
    "arr[0] = values\n",
    "arr\n",
    "arr[:, arr[0].argsort()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "first_name = np.array(['Bob', 'Jane', 'Steve', 'Bill', 'Barbara'])\n",
    "last_name = np.array(['Jones', 'Arnold', 'Arnold', 'Jones', 'Walters'])\n",
    "sorter = np.lexsort((first_name, last_name))\n",
    "sorter\n",
    "list(zip(last_name[sorter], first_name[sorter]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "values = np.array(['2:first', '2:second', '1:first', '1:second',\n",
    "                   '1:third'])\n",
    "key = np.array([2, 2, 1, 1, 1])\n",
    "indexer = key.argsort(kind='mergesort')\n",
    "indexer\n",
    "values.take(indexer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "rng = np.random.default_rng(12345)\n",
    "arr = rng.standard_normal(20)\n",
    "arr\n",
    "np.partition(arr, 3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "indices = np.argpartition(arr, 3)\n",
    "indices\n",
    "arr.take(indices)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "arr = np.array([0, 1, 7, 12, 15])\n",
    "arr.searchsorted(9)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "arr.searchsorted([0, 8, 11, 16])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "arr = np.array([0, 0, 0, 1, 1, 1, 1])\n",
    "arr.searchsorted([0, 1])\n",
    "arr.searchsorted([0, 1], side='right')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = np.floor(rng.uniform(0, 10000, size=50))\n",
    "bins = np.array([0, 100, 1000, 5000, 10000])\n",
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "labels = bins.searchsorted(data)\n",
    "labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.Series(data).groupby(labels).mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "def mean_distance(x, y):\n",
    "    nx = len(x)\n",
    "    result = 0.0\n",
    "    count = 0\n",
    "    for i in range(nx):\n",
    "        result += x[i] - y[i]\n",
    "        count += 1\n",
    "    return result / count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "mmap = np.memmap('mymmap', dtype='float64', mode='w+',\n",
    "                 shape=(10000, 10000))\n",
    "mmap"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "section = mmap[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "section[:] = rng.standard_normal((5, 10000))\n",
    "mmap.flush()\n",
    "mmap\n",
    "del mmap"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "mmap = np.memmap('mymmap', dtype='float64', shape=(10000, 10000))\n",
    "mmap"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "%xdel mmap\n",
    "!rm mymmap"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [],
   "source": [
    "arr_c = np.ones((100, 10000), order='C')\n",
    "arr_f = np.ones((100, 10000), order='F')\n",
    "arr_c.flags\n",
    "arr_f.flags\n",
    "arr_f.flags.f_contiguous"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [],
   "source": [
    "%timeit arr_c.sum(1)\n",
    "%timeit arr_f.sum(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "arr_f.copy('C').flags"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [],
   "source": [
    "arr_c[:50].flags.contiguous\n",
    "arr_c[:, :50].flags"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "%xdel arr_c\n",
    "%xdel arr_f"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.options.display.max_rows = PREVIOUS_MAX_ROWS"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}