make_network.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Creating Folkersma joint network layer\n",
    "\n",
    "Assuming that the network data we received from Niras and Folkersma are in the folders `data/NIRAS` and `data/FOLKERSMA`, respectively (from Dropbox [here for Niras](https://www.dropbox.com/scl/fo/76ome1g3nqpr71akrifba/h?rlkey=3vijuusrgwil3wprd0gmfajhj&dl=0) and [here for Folkersma]())\n",
    "\n",
    "This script imports nodes and edges from both sources and merges them (one file for all nodes; one file for all edges)\n",
    "\n",
    "Run from main repo folder!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "import geopandas as gpd\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import contextily as cx\n",
    "from collections import Counter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# read in Niras data\n",
    "edges1 = gpd.read_file(\"data/NIRAS/straekninger.gpkg\")\n",
    "nodes1 = gpd.read_file(\"data/NIRAS/knudepunkter.gpkg\")\n",
    "\n",
    "# read in Folkersma data\n",
    "edges2 = gpd.read_file(\"data/FOLKERSMA/stretch.shp\")\n",
    "nodes2 = gpd.read_file(\"data/FOLKERSMA/node.shp\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "# check that crs is the same for all data sets\n",
    "assert all([edges1.crs, edges2.crs, nodes1.crs, nodes2.crs])\n",
    "assert edges1.crs == edges2.crs\n",
    "assert nodes1.crs == nodes2.crs\n",
    "assert edges1.crs == nodes1.crs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "# make sure we have the same columns present in both data sets\n",
    "assert sorted(list(edges1.columns)) == sorted(list(edges2.columns))\n",
    "assert sorted(list(nodes1.columns)) == sorted(list(nodes2.columns))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "# make sure data sets don't intersect\n",
    "assert edges1.unary_union.intersection(edges2.unary_union).is_empty\n",
    "assert nodes1.unary_union.intersection(nodes2.unary_union).is_empty"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "# merge the two data frames\n",
    "nodes = pd.concat([nodes1, nodes2], join = \"outer\", ignore_index=True)\n",
    "edges = pd.concat([edges1, edges2], join = \"outer\", ignore_index=True)\n",
    "assert len(nodes.columns) == len(nodes1.columns)\n",
    "assert len(edges.columns) == len(edges1.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "# explode geometries\n",
    "nodes = nodes.explode(ignore_index=True)\n",
    "edges = edges.explode(ignore_index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Correcting child/parent node relations\n",
    "\n",
    "* Some nodes are signalled as main, but have a refmain node that is different from their node ID\n",
    "    * if refmain exists: set ismain to 0\n",
    "    * if remain doesn't exist: ? (not the case here)\n",
    "* Some nodes are signalled as children, but their parent node ID does not exist in the node ID list. \n",
    "    * set refmain to NA\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "# \"ismain\" nodes where refmain and id do not coincide\n",
    "\n",
    "# get index of nodes that shouldn't be main\n",
    "set_ismain_to_0 = nodes[(nodes.ismain == 1) & nodes.refmain.isin(list(nodes.id))].index\n",
    "\n",
    "# check that their refmain node id exists\n",
    "assert all([id in list(nodes.id) for id in list(int(id) for id in nodes.loc[set_ismain_to_0][\"refmain\"])])\n",
    "\n",
    "# set their refmain to 0\n",
    "nodes.loc[set_ismain_to_0, \"ismain\"] = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "# children whose parents do not exist in our data set: set their refmain to NAN\n",
    "set_refmain_to_na = nodes[(nodes.ismain == 0) & nodes.refmain.notna() & -nodes.refmain.isin(list(nodes.id))].index\n",
    "nodes.loc[set_refmain_to_na, \"refmain\"] = np.nan"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "# nodes who are indicated as main, but whose refmain (that shouldnt be there to begin with) \n",
    "# doesn't exist in the node id list\n",
    "set_refmain_to_na = nodes[(nodes.ismain == 1) & nodes.refmain.notna() & -nodes.refmain.isin(list(nodes.id))].index\n",
    "nodes.loc[set_refmain_to_na, \"refmain\"] = np.nan"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "# children whose refmains are faulty\n",
    "# faulty refmains found manually, e.g. nodes are too far away from each other\n",
    "faulty_refmains = [1715, 1822, 4657]\n",
    "for faulty_refmain in faulty_refmains:\n",
    "    nodes.loc[nodes.refmain == faulty_refmain, \"refmain\"] = np.nan"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "# faulty_refmain = 1715\n",
    "# fig, ax = plt.subplots(1,1, figsize = (20,20))\n",
    "# nodes[nodes.refmain == faulty_refmain].plot(ax=ax)\n",
    "# nodes[nodes.id == faulty_refmain].plot(ax=ax, color = \"red\")\n",
    "# cx.add_basemap(ax=ax, crs = nodes.crs, source = cx.providers.CartoDB.Voyager)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**save to file**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# save as gpkgs\n",
    "nodes.to_file(\"data/raw/network/nodes.gpkg\")\n",
    "edges.to_file(\"data/raw/network/edges.gpkg\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "knupu",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}