inital check in for generic mirroring

data-warehouse/notebooks/create-llist-delete-warehouse.ipynb

@@ -1 +1,224 @@
-{"cells":[{"cell_type":"code","source":["# details from Article : https://learn.microsoft.com/en-us/fabric/data-warehouse/collation\n","# default collation is Latin1_General_100_BIN2_UTF8\n","# new collation is Latin1_General_100_CI_AS_KS_WS_SC_UTF8\n","\n","#REST API : https://learn.microsoft.com/en-us/rest/api/fabric/warehouse/items/create-warehouse?tabs=HTTP\n","\n","#sempy version 0.4.0 or higher\n","!pip install semantic-link --q \n","import json\n","import sempy.fabric as fabric\n","from sempy.fabric.exceptions import FabricHTTPException, WorkspaceNotFoundException\n","\n","workspace_id=spark.conf.get(\"trident.workspace.id\")\n","\n","#Instantiate the client\n","client = fabric.FabricRestClient()\n","\n","uri = f\"https://api.fabric.microsoft.com/v1/workspaces/{workspace_id}/items\"\n","payload = { \n","  \"type\": \"Warehouse\", \n","  \"displayName\": \"marktest\", \n","  \"description\": \"New warehouse with case-insensitive collation\", \n","  \"creationPayload\": { \n","    \"defaultCollation\": \"Latin1_General_100_CI_AS_KS_WS_SC_UTF8\" \n","  } \n","}\n","\n","# Call the REST API\n","response = client.post(uri,json= payload)\n","display(response)\n","\n","#data = json.loads(response.text)\n","#display(data)"],"outputs":[{"output_type":"display_data","data":{"application/vnd.livy.statement-meta+json":{"spark_pool":null,"statement_id":74,"statement_ids":[74],"state":"finished","livy_statement_state":"available","session_id":"65aa4f49-978d-45fd-b906-01addd1f27e9","normalized_state":"finished","queued_time":"2024-11-30T12:09:02.1534597Z","session_start_time":null,"execution_start_time":"2024-11-30T12:09:06.9337526Z","execution_finish_time":"2024-11-30T12:09:11.7107053Z","parent_msg_id":"0f55a180-bb20-4af1-ba70-00fa37c019a0"},"text/plain":"StatementMeta(, 65aa4f49-978d-45fd-b906-01addd1f27e9, 74, Finished, Available, Finished)"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"<Response [202]>"},"metadata":{}}],"execution_count":72,"metadata":{"microsoft":{"language":"python","language_group":"synapse_pyspark"}},"id":"64c0e7fb-eff6-4544-aa5f-2b80354243ae"},{"cell_type":"code","source":["# RESPI API : https://learn.microsoft.com/en-us/rest/api/fabric/warehouse/items/list-warehouses?tabs=HTTP\n","\n","import json\n","import sempy.fabric as fabric\n","from sempy.fabric.exceptions import FabricHTTPException, WorkspaceNotFoundException\n","import time\n","target_displayname = 'marktest'\n","\n","workspace_id=spark.conf.get(\"trident.workspace.id\")\n","\n","#Instantiate the client\n","client = fabric.FabricRestClient()\n","\n","statusuri = f\"https://api.fabric.microsoft.com/v1/workspaces/{workspace_id}/warehouses\"\n","matching_id = None\n","\n","while(matching_id is None):\n","    statusresponsedata = client.get(statusuri).json()\n","    datar = statusresponsedata['value']\n","    for item in datar:\n","        whName = item['displayName']\n","        if whName == target_displayname:\n","            matching_id = item['id']\n","            break\n","    \n","    display(\"Waiting....\")\n","    time.sleep(1)\n","\n","display(f\"Warehouse id is {matching_id}\")\n","display(\"Warehouse details:\")\n","print(item)"],"outputs":[],"execution_count":null,"metadata":{"jupyter":{"source_hidden":false,"outputs_hidden":false},"nteract":{"transient":{"deleting":false}},"microsoft":{"language":"python","language_group":"synapse_pyspark"},"collapsed":false},"id":"41e98e7c-42fc-4521-b7e5-6db5a1656143"},{"cell_type":"code","source":["# RESPI API : https://learn.microsoft.com/en-us/rest/api/fabric/warehouse/items/get-warehouse?tabs=HTTP\n","\n","import json\n","import sempy.fabric as fabric\n","from sempy.fabric.exceptions import FabricHTTPException, WorkspaceNotFoundException\n","\n","workspace_id=spark.conf.get(\"trident.workspace.id\")\n","\n","#Instantiate the client\n","client = fabric.FabricRestClient()\n","\n","#matchind_id = 'bd3bb97e-8255-4b33-8ac2-8f63ec53fd23' \n","\n","statusuri = f\"https://api.fabric.microsoft.com/v1/workspaces/{workspace_id}/warehouses/{matching_id}\"\n","\n","statusresponsedata = client.get(statusuri).json()\n","display(statusresponsedata)\n","\n"],"outputs":[],"execution_count":null,"metadata":{"jupyter":{"source_hidden":false,"outputs_hidden":false},"nteract":{"transient":{"deleting":false}},"microsoft":{"language":"python","language_group":"synapse_pyspark"}},"id":"49901dcb-bdd1-4036-920c-42bccf023638"},{"cell_type":"code","source":["# REST API: https://learn.microsoft.com/en-us/rest/api/fabric/warehouse/items/delete-warehouse?tabs=HTTP\n","\n","import json\n","import sempy.fabric as fabric\n","from sempy.fabric.exceptions import FabricHTTPException, WorkspaceNotFoundException\n","\n","workspace_id=spark.conf.get(\"trident.workspace.id\")\n","#Instantiate the client\n","client = fabric.FabricRestClient()\n","\n","uri = f\"https://api.fabric.microsoft.com/v1/workspaces/{workspace_id}/warehouses/{matching_id}\"\n","\n","\n","# Call the REST API\n","response = client.delete(uri)\n","display(response)\n"],"outputs":[],"execution_count":null,"metadata":{"jupyter":{"source_hidden":false,"outputs_hidden":false},"nteract":{"transient":{"deleting":false}},"microsoft":{"language":"python","language_group":"synapse_pyspark"}},"id":"0c9a011d-e4c9-4365-8f93-9cc3927d768e"}],"metadata":{"kernel_info":{"name":"synapse_pyspark"},"kernelspec":{"name":"synapse_pyspark","language":"Python","display_name":"Synapse PySpark"},"language_info":{"name":"python"},"microsoft":{"language":"python","language_group":"synapse_pyspark","ms_spell_check":{"ms_spell_check_language":"en"}},"nteract":{"version":"[email protected]"},"spark_compute":{"compute_id":"/trident/default","session_options":{"conf":{"spark.synapse.nbs.session.timeout":"1200000"}}},"synapse_widget":{"version":"0.1","state":{}},"dependencies":{}},"nbformat":4,"nbformat_minor":5}
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "64c0e7fb-eff6-4544-aa5f-2b80354243ae",
+   "metadata": {
+    "microsoft": {
+     "language": "python",
+     "language_group": "synapse_pyspark"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "# details from Article : https://learn.microsoft.com/en-us/fabric/data-warehouse/collation\n",
+    "# default collation is Latin1_General_100_BIN2_UTF8\n",
+    "# new collation is Latin1_General_100_CI_AS_KS_WS_SC_UTF8\n",
+    "\n",
+    "#REST API : https://learn.microsoft.com/en-us/rest/api/fabric/warehouse/items/create-warehouse?tabs=HTTP\n",
+    "\n",
+    "#sempy version 0.4.0 or higher\n",
+    "!pip install semantic-link --q \n",
+    "import json\n",
+    "import sempy.fabric as fabric\n",
+    "from sempy.fabric.exceptions import FabricHTTPException, WorkspaceNotFoundException\n",
+    "\n",
+    "workspace_id=spark.conf.get(\"trident.workspace.id\")\n",
+    "\n",
+    "#Instantiate the client\n",
+    "client = fabric.FabricRestClient()\n",
+    "\n",
+    "uri = f\"https://api.fabric.microsoft.com/v1/workspaces/{workspace_id}/items\"\n",
+    "payload = { \n",
+    "  \"type\": \"Warehouse\", \n",
+    "  \"displayName\": \"marktest\", \n",
+    "  \"description\": \"New warehouse with case-insensitive collation\", \n",
+    "  \"creationPayload\": { \n",
+    "    \"defaultCollation\": \"Latin1_General_100_CI_AS_KS_WS_SC_UTF8\" \n",
+    "  } \n",
+    "}\n",
+    "\n",
+    "# Call the REST API\n",
+    "response = client.post(uri,json= payload)\n",
+    "display(response)\n",
+    "\n",
+    "#data = json.loads(response.text)\n",
+    "#display(data)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "41e98e7c-42fc-4521-b7e5-6db5a1656143",
+   "metadata": {
+    "collapsed": false,
+    "jupyter": {
+     "outputs_hidden": false,
+     "source_hidden": false
+    },
+    "microsoft": {
+     "language": "python",
+     "language_group": "synapse_pyspark"
+    },
+    "nteract": {
+     "transient": {
+      "deleting": false
+     }
+    }
+   },
+   "outputs": [],
+   "source": [
+    "# RESPI API : https://learn.microsoft.com/en-us/rest/api/fabric/warehouse/items/list-warehouses?tabs=HTTP\n",
+    "\n",
+    "import json\n",
+    "import sempy.fabric as fabric\n",
+    "from sempy.fabric.exceptions import FabricHTTPException, WorkspaceNotFoundException\n",
+    "import time\n",
+    "target_displayname = 'marktest'\n",
+    "\n",
+    "workspace_id=spark.conf.get(\"trident.workspace.id\")\n",
+    "\n",
+    "#Instantiate the client\n",
+    "client = fabric.FabricRestClient()\n",
+    "\n",
+    "statusuri = f\"https://api.fabric.microsoft.com/v1/workspaces/{workspace_id}/warehouses\"\n",
+    "matching_id = None\n",
+    "\n",
+    "while(matching_id is None):\n",
+    "    statusresponsedata = client.get(statusuri).json()\n",
+    "    datar = statusresponsedata['value']\n",
+    "    for item in datar:\n",
+    "        whName = item['displayName']\n",
+    "        if whName == target_displayname:\n",
+    "            matching_id = item['id']\n",
+    "            break\n",
+    "    \n",
+    "    display(\"Waiting....\")\n",
+    "    time.sleep(1)\n",
+    "\n",
+    "display(f\"Warehouse id is {matching_id}\")\n",
+    "display(\"Warehouse details:\")\n",
+    "print(item)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "49901dcb-bdd1-4036-920c-42bccf023638",
+   "metadata": {
+    "jupyter": {
+     "outputs_hidden": false,
+     "source_hidden": false
+    },
+    "microsoft": {
+     "language": "python",
+     "language_group": "synapse_pyspark"
+    },
+    "nteract": {
+     "transient": {
+      "deleting": false
+     }
+    }
+   },
+   "outputs": [],
+   "source": [
+    "# RESPI API : https://learn.microsoft.com/en-us/rest/api/fabric/warehouse/items/get-warehouse?tabs=HTTP\n",
+    "\n",
+    "import json\n",
+    "import sempy.fabric as fabric\n",
+    "from sempy.fabric.exceptions import FabricHTTPException, WorkspaceNotFoundException\n",
+    "\n",
+    "workspace_id=spark.conf.get(\"trident.workspace.id\")\n",
+    "\n",
+    "#Instantiate the client\n",
+    "client = fabric.FabricRestClient()\n",
+    "\n",
+    "#matchind_id = 'bd3bb97e-8255-4b33-8ac2-8f63ec53fd23' \n",
+    "\n",
+    "statusuri = f\"https://api.fabric.microsoft.com/v1/workspaces/{workspace_id}/warehouses/{matching_id}\"\n",
+    "\n",
+    "statusresponsedata = client.get(statusuri).json()\n",
+    "display(statusresponsedata)\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "0c9a011d-e4c9-4365-8f93-9cc3927d768e",
+   "metadata": {
+    "jupyter": {
+     "outputs_hidden": false,
+     "source_hidden": false
+    },
+    "microsoft": {
+     "language": "python",
+     "language_group": "synapse_pyspark"
+    },
+    "nteract": {
+     "transient": {
+      "deleting": false
+     }
+    }
+   },
+   "outputs": [],
+   "source": [
+    "# REST API: https://learn.microsoft.com/en-us/rest/api/fabric/warehouse/items/delete-warehouse?tabs=HTTP\n",
+    "\n",
+    "import json\n",
+    "import sempy.fabric as fabric\n",
+    "from sempy.fabric.exceptions import FabricHTTPException, WorkspaceNotFoundException\n",
+    "\n",
+    "workspace_id=spark.conf.get(\"trident.workspace.id\")\n",
+    "#Instantiate the client\n",
+    "client = fabric.FabricRestClient()\n",
+    "\n",
+    "uri = f\"https://api.fabric.microsoft.com/v1/workspaces/{workspace_id}/warehouses/{matching_id}\"\n",
+    "\n",
+    "\n",
+    "# Call the REST API\n",
+    "response = client.delete(uri)\n",
+    "display(response)\n"
+   ]
+  }
+ ],
+ "metadata": {
+  "dependencies": {},
+  "kernel_info": {
+   "name": "synapse_pyspark"
+  },
+  "kernelspec": {
+   "display_name": "Synapse PySpark",
+   "language": "Python",
+   "name": "synapse_pyspark"
+  },
+  "language_info": {
+   "name": "python"
+  },
+  "microsoft": {
+   "language": "python",
+   "language_group": "synapse_pyspark",
+   "ms_spell_check": {
+    "ms_spell_check_language": "en"
+   }
+  },
+  "nteract": {
+   "version": "[email protected]"
+  },
+  "spark_compute": {
+   "compute_id": "/trident/default",
+   "session_options": {
+    "conf": {
+     "spark.synapse.nbs.session.timeout": "1200000"
+    }
+   }
+  },
+  "synapse_widget": {
+   "state": {},
+   "version": "0.1"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

open-mirroring/GenericMirroring/DatabaseConfig.cs

@@ -0,0 +1,119 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Net.Http.Headers;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace SQLMirroring
+{
+    public class DatabaseConfig
+    {
+        public string ConnectionString { get; set; }
+        public string Type { get; set; }
+
+        public string ChangeTrackingEnabled { get; set; }
+
+        public int syncVersion { get; set; }
+        public List<TableConfig> Tables { get; set; }
+
+        public string LocalLocationforTables { get; set; }
+        public string DatabaseName { get; set; }
+        public string ChangeTrackingSQL { get; set; }
+        public string ChangeTrackingTable { get; set; }
+
+        public string Highwatermark { get; set; }
+        public string HighwatermarkSQL { get; set; }
+
+        public string ChangeIncrementalSQL { get; set; }
+
+        public string FullDataExtractQuery { get; set; }
+    }
+
+    public class TableConfig
+    {
+        public string TableName { get; set; }
+        public string SchemaName { get; set; }
+        public string Status { get; set; }
+
+        public DateTime LastUpdate { get; set; }
+        public int SecondsBetweenChecks { get; set; }
+
+        public string KeyColumn { get; set; }
+        public string OtherColumns { get; set; }
+        public string AdditionalColumns { get; set; }
+
+        public string DeltaVersion { get; set; }
+
+        public string SoftDelete { get; set; }
+
+    }
+
+    public class Root
+    {
+        public List<DatabaseConfig> SQLChangeTrackingConfig { get; set; }
+
+        public ExcelConfig ExcelMirroringConfig { get; set; }
+
+        public List<UploadDetails> uploadDetails { get; set; }
+
+        public string LocationLogging { get; set; }
+
+        public AccessConfig AccessMirroringConfig { get; set; }
+
+        public CSVConfig CSVMirroringConfig { get; set; }
+
+        public Gen2Config Gen2MirroringConfig { get; set; }
+
+        public SharepointConfig SharepointMirroringConfig { get; set; }     
+
+        public Root()
+        {
+            uploadDetails = new List<UploadDetails>();
+        }
+    }
+
+    public  class UploadDetails
+    {
+        public string SPN_Application_ID { get; set; }
+        public string SPN_Secret { get; set; }
+        public string SPN_Tenant { get; set; }
+        public string LandingZone { get; set; }
+
+        public string PathtoAZCopy { get; set; }
+    }
+
+    public class ExcelConfig
+    {
+        public string folderToWatch { get; set; }
+        public string outputFolder { get; set; }
+
+    }
+
+    public class CSVConfig
+    {
+        public string folderToWatch { get; set; }
+        public string outputFolder { get; set; }
+
+    }
+
+    public class AccessConfig
+    {
+        public string folderToWatch { get; set; }
+        public string outputFolder { get; set; }
+
+        public Boolean IncludeFolders { get; set; }
+
+    }
+
+    public class Gen2Config
+    {
+        public string ConnectionString { get; set; }
+    }
+
+    public class SharepointConfig
+    {
+
+    }
+
+}

open-mirroring/GenericMirroring/GenericMirroring.csproj

@@ -0,0 +1,30 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net8.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="EPPlus" Version="7.5.3" />
+    <PackageReference Include="Microsoft.Data.SqlClient" Version="5.2.2" />
+    <PackageReference Include="Parquet.Net" Version="5.0.2" />
+    <PackageReference Include="ParquetSharp" Version="16.1.0" />
+    <PackageReference Include="System.Data.OleDb" Version="9.0.1" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <None Update="azcopy.exe">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
+    <None Update="mirrorconfig - Copy.json">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
+    <None Update="mirrorconfig.json">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
+  </ItemGroup>
+
+</Project>