ngrok-tricks.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "ngrok-tricks - SSH into Colab and view tensorboard using ngrok",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "TPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/shawwn/colab-tricks/blob/master/ngrok-tricks.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tPGkLrbHf4m8",
        "colab_type": "text"
      },
      "source": [
        "#Mount your google drive & symlink it to `/drive`"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SAqwoVyIwoj0",
        "colab_type": "code",
        "outputId": "b502c899-40a0-4d19-c680-2f5379d8977c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 125
        }
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive', force_remount=True)\n",
        "\n",
        "!mkdir -p /drive\n",
        "#umount /drive\n",
        "!mount --bind /content/drive/My\\ Drive /drive\n",
        "!mkdir -p /drive/ngrok-ssh\n",
        "!mkdir -p ~/.ssh"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3Aietf%3Awg%3Aoauth%3A2.0%3Aoob&scope=email%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdocs.test%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdrive%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdrive.photos.readonly%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fpeopleapi.readonly&response_type=code\n",
            "\n",
            "Enter your authorization code:\n",
            "··········\n",
            "Mounted at /content/drive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5BfjbdkmgWPz",
        "colab_type": "text"
      },
      "source": [
        "#(optional) If a TPU is attached, print its address\n",
        "\n",
        "This notebook works for all runtime types: CPU, GPU, and TPU.\n",
        "\n",
        "If connected to TPU runtime, this cell prints info about the attached TPU. (This is just for convenience / informational purposes.)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2jKgMEagipmt",
        "colab_type": "code",
        "outputId": "8bd8749b-13f2-4a1a-c280-1f15e904a568",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 247
        }
      },
      "source": [
        "import os\n",
        "import pprint\n",
        "import tensorflow as tf\n",
        "\n",
        "if 'COLAB_TPU_ADDR' not in os.environ:\n",
        "  print('Not connected to a TPU runtime')\n",
        "else:\n",
        "  tpu_address = 'grpc://' + os.environ['COLAB_TPU_ADDR']\n",
        "  print ('TPU address is', tpu_address)\n",
        "  with tf.Session(tpu_address) as session:\n",
        "    devices = session.list_devices()\n",
        "  print('TPU devices:')\n",
        "  pprint.pprint(devices)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "TPU address is grpc://10.44.120.90:8470\n",
            "TPU devices:\n",
            "[_DeviceAttributes(/job:tpu_worker/replica:0/task:0/device:CPU:0, CPU, -1, 1173016086274077236),\n",
            " _DeviceAttributes(/job:tpu_worker/replica:0/task:0/device:XLA_CPU:0, XLA_CPU, 17179869184, 2991867235668810802),\n",
            " _DeviceAttributes(/job:tpu_worker/replica:0/task:0/device:TPU:0, TPU, 17179869184, 6184859761337604570),\n",
            " _DeviceAttributes(/job:tpu_worker/replica:0/task:0/device:TPU:1, TPU, 17179869184, 2958198024988600890),\n",
            " _DeviceAttributes(/job:tpu_worker/replica:0/task:0/device:TPU:2, TPU, 17179869184, 7018531718559331614),\n",
            " _DeviceAttributes(/job:tpu_worker/replica:0/task:0/device:TPU:3, TPU, 17179869184, 4577201967883118425),\n",
            " _DeviceAttributes(/job:tpu_worker/replica:0/task:0/device:TPU:4, TPU, 17179869184, 1807161912338548618),\n",
            " _DeviceAttributes(/job:tpu_worker/replica:0/task:0/device:TPU:5, TPU, 17179869184, 6466936639391228903),\n",
            " _DeviceAttributes(/job:tpu_worker/replica:0/task:0/device:TPU:6, TPU, 17179869184, 5609808055865915066),\n",
            " _DeviceAttributes(/job:tpu_worker/replica:0/task:0/device:TPU:7, TPU, 17179869184, 15329956649730022893),\n",
            " _DeviceAttributes(/job:tpu_worker/replica:0/task:0/device:TPU_SYSTEM:0, TPU_SYSTEM, 8589934592, 8355276426596913199)]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "q6DkHRhXnAZE",
        "colab_type": "text"
      },
      "source": [
        "# Install `sshd`, `tmux`, and `vim`\n",
        "\n",
        "You can add any other prerequisites you want, e.g. `emacs`. That way, when you ssh into the colab instance, all of these will already be installed.\n",
        "\n",
        "(You'd otherwise have to run `apt-get install emacs` every time you ssh into a fresh colab instance. It's easier just to install all your favorite programs here, ahead of time.)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "J_WEZOLUyTs2",
        "colab_type": "code",
        "colab": {},
        "cellView": "code"
      },
      "source": [
        "!apt-get install ssh tmux vim"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "aqhQAOrJnvkq",
        "colab_type": "text"
      },
      "source": [
        "# Create an sshd configuration file\n",
        "\n",
        "- Normally, you shouldn't need to modify this next cell; the configuration is designed to work with colab.\n",
        "\n",
        "Simply run this cell:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0v5HMu_nfROT",
        "colab_type": "code",
        "outputId": "bf13dfd0-23cc-455d-ba39-7300cf171725",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "cellView": "form"
      },
      "source": [
        "#@title\n",
        "%%writefile /drive/ngrok-ssh/sshd_config\n",
        "#\t$OpenBSD: sshd_config,v 1.101 2017/03/14 07:19:07 djm Exp $\n",
        "\n",
        "# This is the sshd server system-wide configuration file.  See\n",
        "# sshd_config(5) for more information.\n",
        "\n",
        "# This sshd was compiled with PATH=/usr/bin:/bin:/usr/sbin:/sbin\n",
        "\n",
        "# The strategy used for options in the default sshd_config shipped with\n",
        "# OpenSSH is to specify options with their default value where\n",
        "# possible, but leave them commented.  Uncommented options override the\n",
        "# default value.\n",
        "\n",
        "#Port 22\n",
        "#AddressFamily any\n",
        "#ListenAddress 0.0.0.0\n",
        "#ListenAddress ::\n",
        "\n",
        "#HostKey /etc/ssh/ssh_host_rsa_key\n",
        "#HostKey /etc/ssh/ssh_host_ecdsa_key\n",
        "#HostKey /etc/ssh/ssh_host_ed25519_key\n",
        "\n",
        "# Ciphers and keying\n",
        "#RekeyLimit default none\n",
        "\n",
        "# Logging\n",
        "#SyslogFacility AUTH\n",
        "#LogLevel INFO\n",
        "\n",
        "# Authentication:\n",
        "\n",
        "#LoginGraceTime 2m\n",
        "#PermitRootLogin prohibit-password\n",
        "#StrictModes yes\n",
        "#MaxAuthTries 6\n",
        "#MaxSessions 10\n",
        "\n",
        "#PubkeyAuthentication yes\n",
        "\n",
        "# Expect .ssh/authorized_keys2 to be disregarded by default in future.\n",
        "#AuthorizedKeysFile\t.ssh/authorized_keys .ssh/authorized_keys2\n",
        "\n",
        "#AuthorizedPrincipalsFile none\n",
        "\n",
        "#AuthorizedKeysCommand none\n",
        "#AuthorizedKeysCommandUser nobody\n",
        "\n",
        "# For this to work you will also need host keys in /etc/ssh/ssh_known_hosts\n",
        "#HostbasedAuthentication no\n",
        "# Change to yes if you don't trust ~/.ssh/known_hosts for\n",
        "# HostbasedAuthentication\n",
        "#IgnoreUserKnownHosts no\n",
        "# Don't read the user's ~/.rhosts and ~/.shosts files\n",
        "#IgnoreRhosts yes\n",
        "\n",
        "# To disable tunneled clear text passwords, change to no here!\n",
        "#PasswordAuthentication yes\n",
        "#PermitEmptyPasswords no\n",
        "\n",
        "# Change to yes to enable challenge-response passwords (beware issues with\n",
        "# some PAM modules and threads)\n",
        "ChallengeResponseAuthentication no\n",
        "\n",
        "# Kerberos options\n",
        "#KerberosAuthentication no\n",
        "#KerberosOrLocalPasswd yes\n",
        "#KerberosTicketCleanup yes\n",
        "#KerberosGetAFSToken no\n",
        "\n",
        "# GSSAPI options\n",
        "#GSSAPIAuthentication no\n",
        "#GSSAPICleanupCredentials yes\n",
        "#GSSAPIStrictAcceptorCheck yes\n",
        "#GSSAPIKeyExchange no\n",
        "\n",
        "# Set this to 'yes' to enable PAM authentication, account processing,\n",
        "# and session processing. If this is enabled, PAM authentication will\n",
        "# be allowed through the ChallengeResponseAuthentication and\n",
        "# PasswordAuthentication.  Depending on your PAM configuration,\n",
        "# PAM authentication via ChallengeResponseAuthentication may bypass\n",
        "# the setting of \"PermitRootLogin without-password\".\n",
        "# If you just want the PAM account and session checks to run without\n",
        "# PAM authentication, then enable this but set PasswordAuthentication\n",
        "# and ChallengeResponseAuthentication to 'no'.\n",
        "UsePAM yes\n",
        "\n",
        "#AllowAgentForwarding yes\n",
        "AllowTcpForwarding yes\n",
        "#GatewayPorts no\n",
        "X11Forwarding yes\n",
        "#X11DisplayOffset 10\n",
        "#X11UseLocalhost yes\n",
        "#PermitTTY yes\n",
        "PrintMotd no\n",
        "#PrintLastLog yes\n",
        "#TCPKeepAlive yes\n",
        "#UseLogin no\n",
        "#PermitUserEnvironment no\n",
        "#Compression delayed\n",
        "#ClientAliveInterval 0\n",
        "#ClientAliveCountMax 3\n",
        "#UseDNS no\n",
        "#PidFile /var/run/sshd.pid\n",
        "#MaxStartups 10:30:100\n",
        "#PermitTunnel no\n",
        "#ChrootDirectory none\n",
        "#VersionAddendum none\n",
        "\n",
        "# no default banner path\n",
        "#Banner none\n",
        "\n",
        "# Allow client to pass locale environment variables\n",
        "AcceptEnv LANG LC_*\n",
        "\n",
        "# override default of no subsystems\n",
        "Subsystem\tsftp\t/usr/lib/openssh/sftp-server\n",
        "\n",
        "# Example of overriding settings on a per-user basis\n",
        "#Match User anoncvs\n",
        "#\tX11Forwarding no\n",
        "#\tAllowTcpForwarding no\n",
        "#\tPermitTTY no\n",
        "#\tForceCommand cvs server"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Writing /drive/ngrok-ssh/sshd_config\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YbPbsr2lhd6_",
        "colab_type": "text"
      },
      "source": [
        "# Paste the contents of your local `~/.ssh/id_rsa.pub` file into this next cell\n",
        "\n",
        "- If you don't have an `~/.ssh/id_rsa.pub` file, use `ssh-keygen` to make one.\n",
        "\n",
        "- Remember to actually run the cell after you paste your key into it!\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wmyQ60pehfYu",
        "colab_type": "code",
        "outputId": "8c28b951-f55c-443c-f04d-cbff9a54ab01",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "%%writefile ~/.ssh/authorized_keys\n",
        "REPLACE_THIS_LINE_WITH_YOUR_ID_RSA_PUB_FILE"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Overwriting /root/.ssh/authorized_keys\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Uphyo1i2ldXF",
        "colab_type": "text"
      },
      "source": [
        "# Restart sshd"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CGNbcsZS2EpR",
        "colab_type": "code",
        "outputId": "1f3fff8d-5101-4d26-faa6-4e04521a4d96",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        }
      },
      "source": [
        "!cp /drive/ngrok-ssh/sshd_config /etc/ssh/sshd_config\n",
        "!service ssh restart\n",
        "!mkdir -p ~/.ssh"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            " * Restarting OpenBSD Secure Shell server sshd\n",
            "   ...done.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jBqoQ96tf-GR",
        "colab_type": "text"
      },
      "source": [
        "#Install ngrok"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1B_aC562JbJU",
        "colab_type": "code",
        "outputId": "286cc14d-2218-4ba2-8b7c-b6bbd99a6c8c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 247
        }
      },
      "source": [
        "!mkdir -p /drive/ngrok-ssh\n",
        "%cd /drive/ngrok-ssh\n",
        "!wget https://bin.equinox.io/c/4VmDzA7iaHb/ngrok-stable-linux-amd64.zip -O ngrok-stable-linux-amd64.zip\n",
        "!unzip -u ngrok-stable-linux-amd64.zip\n",
        "!cp /drive/ngrok-ssh/ngrok /ngrok\n",
        "!chmod +x /ngrok"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/drive/ngrok-ssh\n",
            "--2019-11-13 11:34:54--  https://bin.equinox.io/c/4VmDzA7iaHb/ngrok-stable-linux-amd64.zip\n",
            "Resolving bin.equinox.io (bin.equinox.io)... 52.86.66.223, 34.197.213.11, 52.204.38.150, ...\n",
            "Connecting to bin.equinox.io (bin.equinox.io)|52.86.66.223|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 13773305 (13M) [application/octet-stream]\n",
            "Saving to: ‘ngrok-stable-linux-amd64.zip’\n",
            "\n",
            "ngrok-stable-linux- 100%[===================>]  13.13M  30.6MB/s    in 0.4s    \n",
            "\n",
            "2019-11-13 11:34:55 (30.6 MB/s) - ‘ngrok-stable-linux-amd64.zip’ saved [13773305/13773305]\n",
            "\n",
            "Archive:  ngrok-stable-linux-amd64.zip\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "UvBwXHZ5jQ1w",
        "colab_type": "text"
      },
      "source": [
        "# Sign-in with ngrok\n",
        "\n",
        "- Sign up for a free ngrok account: https://ngrok.com\n",
        "\n",
        "- Copy your authtoken from https://dashboard.ngrok.com/auth and paste it into the following cell\n",
        "\n",
        "- Run the cell"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4k-1v9HPhBBG",
        "colab_type": "code",
        "outputId": "fdde4dfd-05f5-4405-c60c-46e80ea7c049",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "!/ngrok authtoken PASTE_YOUR_AUTHTOKEN_HERE"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Authtoken saved to configuration file: /root/.ngrok2/ngrok.yml\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cjn_XRvbjtXm",
        "colab_type": "text"
      },
      "source": [
        "# Create various ngrok tunnel configurations\n",
        "\n",
        "Run each of these cells, one by one:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RyDVTHD2PnRI",
        "colab_type": "code",
        "outputId": "40dab8d3-84ad-4fdb-c6bd-347c150e7825",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "%%writefile /drive/ngrok-ssh/ssh.yml\n",
        "tunnels:\n",
        "  ssh:\n",
        "    proto: tcp\n",
        "    addr: 22"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Writing /drive/ngrok-ssh/ssh.yml\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OewiP50wPzpl",
        "colab_type": "code",
        "outputId": "e6204ba5-97e1-4b78-e35a-1964cab4d03c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "%%writefile /drive/ngrok-ssh/tensorboard.yml\n",
        "tunnels:\n",
        "  tensorboard:\n",
        "    proto: http\n",
        "    addr: 6006\n",
        "    inspect: false\n",
        "    bind_tls: true"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Overwriting /drive/ngrok-ssh/tensorboard.yml\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mcTC95zkRAA7",
        "colab_type": "code",
        "outputId": "c6b36fd0-a1c6-4075-ef1d-40817f876e5c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "%%writefile /drive/ngrok-ssh/http8080.yml\n",
        "tunnels:\n",
        "  http8080:\n",
        "    proto: http\n",
        "    addr: 8080\n",
        "    inspect: false\n",
        "    bind_tls: true"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Overwriting /drive/ngrok-ssh/http8080.yml\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jUBa9tBMTCQK",
        "colab_type": "code",
        "outputId": "5f3b7d71-b539-4af0-eb50-54a65d3c510d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "%%writefile /drive/ngrok-ssh/tcp8080.yml\n",
        "tunnels:\n",
        "  tcp8080:\n",
        "    proto: tcp\n",
        "    addr: 8080"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Writing /drive/ngrok-ssh/tcp8080.yml\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ui8IvOpCQGAD",
        "colab_type": "code",
        "outputId": "31eaadf7-223b-4fcb-98d6-21c9b7b724f4",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "import os\n",
        "if 'COLAB_TPU_ADDR' in os.environ:\n",
        "  with open('/drive/ngrok-ssh/tpu.yml', 'w') as f:\n",
        "    f.write(\"\"\"\n",
        "tunnels:\n",
        "  tpu:\n",
        "    proto: tcp\n",
        "    addr: {}\n",
        "\"\"\".format(os.environ['COLAB_TPU_ADDR']).strip())\n",
        "  print('Wrote /drive/ngrok-ssh/tpu.yml')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Wrote /drive/ngrok-ssh/tpu.yml\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nT0-i3VoKjco",
        "colab_type": "code",
        "outputId": "41ca301e-66ce-4140-846a-077357ab4813",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "%%writefile /drive/ngrok-ssh/run_ngrok.sh\n",
        "#!/bin/sh\n",
        "set -x\n",
        "/ngrok start --config ~/.ngrok2/ngrok.yml --config /drive/ngrok-ssh/ssh.yml --log=stdout --config /drive/ngrok-ssh/tensorboard.yml --config /drive/ngrok-ssh/http8080.yml --config /drive/ngrok-ssh/tcp8080.yml --config /drive/ngrok-ssh/tpu.yml \"$@\""
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Overwriting /drive/ngrok-ssh/run_ngrok.sh\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "oB6qmwwManfJ",
        "colab_type": "text"
      },
      "source": [
        "#Start ngrok"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Xr4-BRInUcEB",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import os\n",
        "if 'COLAB_TPU_ADDR' in os.environ:\n",
        "  get_ipython().system_raw('bash /drive/ngrok-ssh/run_ngrok.sh ssh tensorboard tcp8080 tpu &')\n",
        "else:\n",
        "  get_ipython().system_raw('bash /drive/ngrok-ssh/run_ngrok.sh ssh tensorboard tcp8080 &')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kcj4Q_K9SF4v",
        "colab_type": "code",
        "outputId": "dc772b10-003b-4f6b-dd18-522e70bfee48",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 107
        }
      },
      "source": [
        "!ps aux | grep ngrok"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "root        2345  0.0  0.0  39192  6660 ?        S    10:32   0:00 bash /drive/ngrok-ssh/run_ngrok.sh ssh tensorboard tcp8080 tpu\n",
            "root        2346  0.2  0.1 122768 19936 ?        Sl   10:32   0:01 /ngrok start --config /root/.ngrok2/ngrok.yml --config /drive/ngrok-ssh/ssh.yml --log=stdout --config /drive/ngrok-ssh/tensorboard.yml --config /drive/ngrok-ssh/http8080.yml --config /drive/ngrok-ssh/tcp8080.yml --config /drive/ngrok-ssh/tpu.yml ssh tensorboard tcp8080 tpu\n",
            "root        2599  0.0  0.0  39196  6620 ?        S    10:43   0:00 /bin/bash -c ps aux | grep ngrok\n",
            "root        2601  0.0  0.0  38568  4876 ?        S    10:43   0:00 grep ngrok\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hSou5G_7bfpN",
        "colab_type": "text"
      },
      "source": [
        "# Make an ngrok API\n",
        "Let's define some helper functions:\n",
        "\n",
        "- `get_ngrok_info()` - returns a JSON object containing info about all open ngrok tunnels\n",
        "\n",
        "- `get_ngrok_tunnels()` - enumerate over all open ngrok tunnels. Yields `name, info` tuples.\n",
        "\n",
        "- `get_ngrok_tunnel(name)` - get info about a tunnel\n",
        "\n",
        "- `get_ngrok_url(name, local=False)` - get a tunnel's URL. If `local` is `False`, returns a public URL that anyone can access, otherwise returns the local url being forwarded"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2GOhOETZLfjI",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import requests\n",
        "import urllib.parse\n",
        "\n",
        "def get_ngrok_info():\n",
        "  return requests.get('http://localhost:4040/api/tunnels').json()\n",
        "\n",
        "def get_ngrok_tunnels():\n",
        "  for tunnel in get_ngrok_info()['tunnels']:\n",
        "    name = tunnel['name']\n",
        "    yield name, tunnel\n",
        "\n",
        "def get_ngrok_tunnel(name):\n",
        "  for name1, tunnel in get_ngrok_tunnels():\n",
        "    if name == name1:\n",
        "      return tunnel\n",
        "\n",
        "def get_ngrok_url(name, local=False):\n",
        "  if local:\n",
        "    return get_ngrok_tunnel(name)['config']['addr']\n",
        "  else:\n",
        "    return get_ngrok_tunnel(name)['public_url']"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RS0YENvSbCA2",
        "colab_type": "text"
      },
      "source": [
        "# Skip this section\n",
        "\n",
        "These are various commands for debugging ngrok."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_Nm37gOgT8Fs",
        "colab_type": "code",
        "outputId": "55e00bd5-64e1-499f-bc51-b670bcf5dcd6",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 338
        }
      },
      "source": [
        "#!bash /drive/ngrok-ssh/run_ngrok.sh ssh tensorboard tcp8080 tpu"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "+ /ngrok start --config /root/.ngrok2/ngrok.yml --config /drive/ngrok-ssh/ssh.yml --log=stdout --config /drive/ngrok-ssh/tensorboard.yml --config /drive/ngrok-ssh/http8080.yml --config /drive/ngrok-ssh/tcp8080.yml --config /drive/ngrok-ssh/tpu.yml ssh tensorboard tcp8080 tpu\n",
            "\u001b[32mINFO\u001b[0m[11-13|10:31:21] open config file                         \u001b[32mpath\u001b[0m=/root/.ngrok2/ngrok.yml \u001b[32merr\u001b[0m=nil\n",
            "\u001b[32mINFO\u001b[0m[11-13|10:31:21] open config file                         \u001b[32mpath\u001b[0m=/drive/ngrok-ssh/ssh.yml \u001b[32merr\u001b[0m=nil\n",
            "\u001b[32mINFO\u001b[0m[11-13|10:31:21] open config file                         \u001b[32mpath\u001b[0m=/drive/ngrok-ssh/tensorboard.yml \u001b[32merr\u001b[0m=nil\n",
            "\u001b[32mINFO\u001b[0m[11-13|10:31:21] open config file                         \u001b[32mpath\u001b[0m=/drive/ngrok-ssh/http8080.yml \u001b[32merr\u001b[0m=nil\n",
            "\u001b[32mINFO\u001b[0m[11-13|10:31:21] open config file                         \u001b[32mpath\u001b[0m=/drive/ngrok-ssh/tcp8080.yml \u001b[32merr\u001b[0m=nil\n",
            "\u001b[32mINFO\u001b[0m[11-13|10:31:21] open config file                         \u001b[32mpath\u001b[0m=/drive/ngrok-ssh/tpu.yml \u001b[32merr\u001b[0m=nil\n",
            "t=2019-11-13T10:31:21+0000 lvl=info msg=\"starting web service\" obj=web addr=127.0.0.1:4040\n",
            "t=2019-11-13T10:31:21+0000 lvl=info msg=\"tunnel session started\" obj=tunnels.session\n",
            "t=2019-11-13T10:31:21+0000 lvl=info msg=\"client session established\" obj=csess id=b9264609890c\n",
            "t=2019-11-13T10:31:21+0000 lvl=info msg=\"started tunnel\" obj=tunnels name=ssh addr=//localhost:22 url=tcp://0.tcp.ngrok.io:10936\n",
            "t=2019-11-13T10:31:21+0000 lvl=info msg=\"started tunnel\" obj=tunnels name=tpu addr=//10.44.120.90:8470 url=tcp://0.tcp.ngrok.io:18574\n",
            "t=2019-11-13T10:31:21+0000 lvl=info msg=\"started tunnel\" obj=tunnels name=tcp8080 addr=//localhost:8080 url=tcp://0.tcp.ngrok.io:13225\n",
            "t=2019-11-13T10:31:21+0000 lvl=info msg=\"started tunnel\" obj=tunnels name=tensorboard addr=http://localhost:6006 url=https://816fecdb.ngrok.io\n",
            "t=2019-11-13T10:31:35+0000 lvl=info msg=\"received stop request\" obj=app stopReq=\"{err:<nil> restart:false}\"\n",
            "t=2019-11-13T10:31:35+0000 lvl=info msg=\"session closing\" obj=tunnels.session err=nil\n",
            "t=2019-11-13T10:31:35+0000 lvl=info msg=\"accept failed\" obj=csess id=b9264609890c err=\"reconnecting session closed\"\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Vr6wsoUyKojz",
        "colab_type": "code",
        "outputId": "3a699985-ca7f-42dc-e5ee-61d5d72b2d6a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "#!curl -s http://localhost:4040/api/tunnels  | python3 -c \"import sys, json; url=json.load(sys.stdin)['tunnels'][0]['public_url']; import urllib.parse; print(urllib.parse.urlparse(url).netloc)\""
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "0.tcp.ngrok.io:10596\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-mEHcV72YTbC",
        "colab_type": "text"
      },
      "source": [
        "#Start Tensorboard"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NjmwmXijNA55",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "LOG_DIR = '/drive/gpt-2-train/checkpoint/run1'\n",
        "get_ipython().system_raw(\n",
        "    'tensorboard --logdir {} --host 0.0.0.0 --port 6006 &'\n",
        "    .format(LOG_DIR)\n",
        ")\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZnWHZoJLNVbs",
        "colab_type": "code",
        "outputId": "223b8e57-0687-45bb-cde9-0fdfe6ed286c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 125
        }
      },
      "source": [
        "!ps aux | grep tensorboard"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "root        1588  0.2  2.2 872976 304868 ?       Sl   10:00   0:06 /usr/bin/python2 /usr/local/bin/tensorboard --logdir /drive/gpt-2-train/checkpoint/run1 --host 0.0.0.0 --port 6006\n",
            "root        2345  0.0  0.0  39192  6660 ?        S    10:32   0:00 bash /drive/ngrok-ssh/run_ngrok.sh ssh tensorboard tcp8080 tpu\n",
            "root        2346  0.2  0.1 122768 19348 ?        Sl   10:32   0:01 /ngrok start --config /root/.ngrok2/ngrok.yml --config /drive/ngrok-ssh/ssh.yml --log=stdout --config /drive/ngrok-ssh/tensorboard.yml --config /drive/ngrok-ssh/http8080.yml --config /drive/ngrok-ssh/tcp8080.yml --config /drive/ngrok-ssh/tpu.yml ssh tensorboard tcp8080 tpu\n",
            "root        2500  0.0  0.0  39192  6624 ?        S    10:39   0:00 /bin/bash -c ps aux | grep tensorboard\n",
            "root        2502  0.0  0.0  38572  5684 ?        S    10:39   0:00 grep tensorboard\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "itCZCppEYd3l",
        "colab_type": "text"
      },
      "source": [
        "# How to view tensorboard"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cdVQ9TDrZONX",
        "colab_type": "code",
        "outputId": "9cb97925-2b37-464a-ca08-03dc7252e238",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "print('To view tensorboard, visit', get_ngrok_url('tensorboard'))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "To view tensorboard, visit https://1f6310e7.ngrok.io\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9P8a-UgLZa3_",
        "colab_type": "text"
      },
      "source": [
        "# Display each ngrok URL\n",
        "\n",
        "For each ngrok tunnel, this cell prints its name, its public URL, and the local URL being forwarded to that public URL."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "d28SLwBxZd4F",
        "colab_type": "code",
        "outputId": "3b2af0da-43b6-4c9f-c334-c918b32e94e7",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 87
        }
      },
      "source": [
        "for name, tunnel in get_ngrok_tunnels():\n",
        "  local_url = get_ngrok_url(name, local=True)\n",
        "  public_url = get_ngrok_url(name, local=False)\n",
        "  print('{:12s} {} <-> {}'.format(name, public_url, local_url))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tpu          tcp://0.tcp.ngrok.io:16601 <-> 10.44.120.90:8470\n",
            "ssh          tcp://0.tcp.ngrok.io:10596 <-> localhost:22\n",
            "tcp8080      tcp://0.tcp.ngrok.io:10056 <-> localhost:8080\n",
            "tensorboard  https://1f6310e7.ngrok.io <-> http://localhost:6006\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Y8_955M4ZGXX",
        "colab_type": "text"
      },
      "source": [
        "# How to SSH into this Colab instance"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BM-qgXeJWUb4",
        "colab_type": "code",
        "outputId": "9f32ef56-a387-4f6d-8f3a-ac4338b05598",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 87
        }
      },
      "source": [
        "print('')\n",
        "public_url, public_port = urllib.parse.urlparse(get_ngrok_url('ssh')).netloc.split(':')\n",
        "print('To SSH into this colab instance, run the following command on your local machine:')\n",
        "print('')\n",
        "\n",
        "print('  ssh -p {} root@{}'.format(public_port, public_url))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n",
            "To SSH into this colab instance, run the following command on your local machine:\n",
            "\n",
            "  ssh -p 10596 root@0.tcp.ngrok.io\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}