samples/visualizing-linked-table-dataflows/visualizing-linked-table-dataflows.ipynb

{"cells":[{"cell_type":"code","source":["%pip install semantic-link-labs"],"outputs":[],"execution_count":null,"metadata":{"jupyter":{"source_hidden":false,"outputs_hidden":true},"nteract":{"transient":{"deleting":false}},"microsoft":{"language":"python","language_group":"synapse_pyspark"}},"id":"030322da-0579-4b3e-a7c7-0f97a59d8bd6"},{"cell_type":"code","source":["# import sempy_labs as labs\n","import sempy_labs as labs\n","\n","# import pandas as pd\n","import pandas as pd"],"outputs":[],"execution_count":null,"metadata":{"jupyter":{"source_hidden":false,"outputs_hidden":false},"nteract":{"transient":{"deleting":false}},"microsoft":{"language":"python","language_group":"synapse_pyspark"},"collapsed":true},"id":"d5b85714-54a1-4ad0-a2d1-6862af2a6974"},{"cell_type":"code","source":["# This function lists all dataflows in a specified workspace. \n","# If no workspace is specified, it uses the current workspace by default.\n","\n","labs.list_dataflows()"],"outputs":[],"execution_count":null,"metadata":{"jupyter":{"source_hidden":false,"outputs_hidden":false},"nteract":{"transient":{"deleting":false}},"microsoft":{"language":"python","language_group":"synapse_pyspark"}},"id":"7d01de9a-432a-413d-9eb4-7056156a84b9"},{"cell_type":"code","source":["# The name of the dataflow for which to find upstream dependencies.\n","df_name = ''\n","\n","# This function looks up all the upstream dataflows for a specified dataflow.\n","df_lineage = labs.list_upstream_dataflows(df_name)\n","\n","df_lineage"],"outputs":[],"execution_count":null,"metadata":{"jupyter":{"source_hidden":false,"outputs_hidden":false},"nteract":{"transient":{"deleting":false}},"microsoft":{"language":"python","language_group":"synapse_pyspark"}},"id":"8fa18162-d245-48bc-8bcf-efeaf56f8bbd"},{"cell_type":"code","source":["import graphviz\n","from graphviz import Digraph\n","from IPython.display import display, Image\n","\n","# Assuming df_lineage is your dataframe\n","df = df_lineage\n","\n","# Create a new directed graph\n","dot = Digraph()\n","\n","# Create a set to keep track of added edges\n","added_edges = set()\n","\n","# Add nodes and edges based on dataframe columns\n","for index, row in df.iterrows():\n","    # Add nodes with correct labels\n","    dot.node(row['Dataflow Id'], label=row['Dataflow Name'], shape='cylinder')\n","    dot.node(row['Upstream Dataflow Id'], label=row['Upstream Dataflow Name'], shape='cylinder')\n","    dot.node(row['Workspace Id'], label=row['Workspace Name'], shape='folder')\n","    dot.node(row['Upstream Workspace Id'], label=row['Upstream Workspace Name'], shape='folder')\n","    \n","    # Add edges for dataflow dependencies\n","    edge = (row['Dataflow Id'], row['Upstream Dataflow Id'])\n","    if edge not in added_edges:\n","        dot.edge(*edge)\n","        added_edges.add(edge)\n","    \n","    # Add edges for workspace dependencies\n","    edge = (row['Workspace Id'], row['Upstream Workspace Id'])\n","    if edge not in added_edges:\n","        dot.edge(*edge)\n","        added_edges.add(edge)\n","    \n","    # Add edges to show which dataflows are in which workspaces\n","    edge = (row['Workspace Id'], row['Dataflow Id'])\n","    if edge not in added_edges:\n","        dot.edge(*edge, style='solid')\n","        added_edges.add(edge)\n","    \n","    edge = (row['Upstream Workspace Id'], row['Upstream Dataflow Id'])\n","    if edge not in added_edges:\n","        dot.edge(*edge, style='solid')\n","        added_edges.add(edge)\n","\n","# Render the graph to a file and display it in the notebook\n","output_path = 'dependency_graph'\n","dot.render(output_path, format='png')\n","display(Image(filename=f'{output_path}.png'))"],"outputs":[],"execution_count":null,"metadata":{"jupyter":{"source_hidden":false,"outputs_hidden":false},"nteract":{"transient":{"deleting":false}},"microsoft":{"language":"python","language_group":"synapse_pyspark"}},"id":"b4a5ef6b-c176-48ae-a66f-873126c29e47"}],"metadata":{"kernel_info":{"name":"synapse_pyspark"},"kernelspec":{"name":"synapse_pyspark","language":"Python","display_name":"Synapse PySpark"},"language_info":{"name":"python"},"microsoft":{"language":"python","language_group":"synapse_pyspark","ms_spell_check":{"ms_spell_check_language":"en"}},"widgets":{},"nteract":{"version":"nteract-front-end@1.0.0"},"spark_compute":{"compute_id":"/trident/default","session_options":{"conf":{"spark.synapse.nbs.session.timeout":"1200000"}}},"dependencies":{}},"nbformat":4,"nbformat_minor":5}