WIP: Testing out using Polaris Docker image in Integration Test Suite

dev/docker-compose-integration.yml

@@ -41,19 +41,14 @@ services:
       - hive:hive
       - minio:minio
   rest:
-    image: tabulario/iceberg-rest
+    build: https://github.com/apache/polaris.git
     container_name: pyiceberg-rest
     networks:
       iceberg_net:
     ports:
       - 8181:8181
-    environment:
-      - AWS_ACCESS_KEY_ID=admin
-      - AWS_SECRET_ACCESS_KEY=password
-      - AWS_REGION=us-east-1
-      - CATALOG_WAREHOUSE=s3://warehouse/
-      - CATALOG_IO__IMPL=org.apache.iceberg.aws.s3.S3FileIO
-      - CATALOG_S3_ENDPOINT=http://minio:9000
+    volumes:
+      - ./warehouse:/warehouse
   minio:
     image: minio/minio
     container_name: pyiceberg-minio

dev/provision.py

@@ -22,17 +22,165 @@
 from pyiceberg.schema import Schema
 from pyiceberg.types import FixedType, NestedField, UUIDType
 
-spark = SparkSession.builder.getOrCreate()
+import json
+
+from requests import HTTPError, Session
+
+PRINCIPAL_TOKEN="principal:root;realm:default-realm"
+POLARIS_URL="http://rest:8181"
+PRINCIPAL_NAME="iceberg"
+CATALOG_NAME="polaris"
+CATALOG_ROLE="admin_role"
+PRINCIPAL_ROLE = "admin_principal_role"
+
+def create_principal(session: Session) -> str:
+    response = session.get(url=f"{POLARIS_URL}/api/management/v1/principals/{PRINCIPAL_NAME}")
+    try:
+        # rotate creds
+        response.raise_for_status()
+        response = session.delete(
+                url=f"{POLARIS_URL}/api/management/v1/principals/{PRINCIPAL_NAME}",
+            )
+    finally:
+        # create principal
+        data = {"principal": {"name": PRINCIPAL_NAME}, "credentialRotationRequired": 'false'}
+        response = session.post(
+                url=f"{POLARIS_URL}/api/management/v1/principals", data=json.dumps(data),
+            )
+    credentials = response.json()["credentials"]
+
+    principal_credential = f"{credentials['clientId']}:{credentials['clientSecret']}"
+    return principal_credential
+
+def create_catalog(session: Session) -> None:
+    response = session.get(
+            url=f"{POLARIS_URL}/api/management/v1/catalogs/{CATALOG_NAME}",
+        )
+    try:
+        response.raise_for_status()
+    except HTTPError:
+        # Create Catalog
+        data = {
+            "catalog": {
+            "name": CATALOG_NAME,
+            "type": "INTERNAL",
+            "readOnly": False,
+            "properties": {
+                "default-base-location": "file:///warehouse"
+            },
+            "storageConfigInfo": {
+                "storageType": "FILE",
+                "allowedLocations": [
+                "file:///warehouse"
+                ]
+            }
+            }
+        }
+        response = session.post(
+                url=f"{POLARIS_URL}/api/management/v1/catalogs", data=json.dumps(data),
+            )
+        response.raise_for_status()
+
+def create_catalog_role(session: Session) -> None:
+    try:
+        response = session.get(
+            url=f"{POLARIS_URL}/api/management/v1/catalogs/{CATALOG_NAME}/catalog-roles/{CATALOG_ROLE}"
+        )
+        response.raise_for_status()
+    except HTTPError:
+        # Create Catalog Role
+        data = {
+            "catalogRole": {
+            "name": CATALOG_ROLE,
+            }
+        }
+        response = session.post(
+                url=f"{POLARIS_URL}/api/management/v1/catalogs/{CATALOG_NAME}/catalog-roles", data=json.dumps(data),
+            )
+        response.raise_for_status()
+
+def grant_catalog_privileges(session: Session) -> None:
+    # Grant Catalog privileges to the catalog role
+    data = {
+        "grant": {
+        "type": "catalog",
+        "privilege": "CATALOG_MANAGE_CONTENT"
+        }
+    }
+    response = session.put(
+            url=f"{POLARIS_URL}/api/management/v1/catalogs/{CATALOG_NAME}/catalog-roles/{CATALOG_ROLE}/grants", data=json.dumps(data),
+        )
+    response.raise_for_status()
+
+def create_principal_role(session: Session) -> None:
+    try:
+        response = session.get(
+            url=f"{POLARIS_URL}/api/management/v1/principal-roles/{PRINCIPAL_ROLE}",
+        )
+        response.raise_for_status()
+    except HTTPError:
+        # Create a principal role
+        data = {
+            "principalRole": {
+            "name": PRINCIPAL_ROLE,
+            }
+        }
+        response = session.post(
+                url=f"{POLARIS_URL}/api/management/v1/principal-roles", data=json.dumps(data),
+            )
+        response.raise_for_status()
+
+    # Assign the catalog role to the principal role
+    data = {
+        "catalogRole": {
+        "name": CATALOG_ROLE,
+        }
+    }
+    response = session.put(
+            url=f"{POLARIS_URL}/api/management/v1/principal-roles/{PRINCIPAL_ROLE}/catalog-roles/{CATALOG_NAME}", data=json.dumps(data),
+        )
+    response.raise_for_status()
+
+    # Assign the principal role to the root principal
+    data = {
+        "principalRole": {
+        "name": PRINCIPAL_ROLE,
+        }
+    }
+    response = session.put(
+            url=f"{POLARIS_URL}/api/management/v1/principals/{PRINCIPAL_NAME}/principal-roles", data=json.dumps(data),
+        )
+    response.raise_for_status()
+
+session = Session()
+session.headers["Content-type"] = "application/json"
+session.headers["Accept"] = "application/json"
+session.headers["Authorization"] = f"Bearer {PRINCIPAL_TOKEN}"
+
+principal_credential = create_principal(session)
+create_catalog(session)
+create_catalog_role(session)
+grant_catalog_privileges(session)
+create_principal_role(session)
+
+spark = SparkSession.builder.config(
+        "spark.sql.catalog.rest.credential", principal_credential
+    ).getOrCreate()
+
+print(spark.sparkContext.getConf().getAll())
 
 catalogs = {
     'rest': load_catalog(
         "rest",
         **{
             "type": "rest",
-            "uri": "http://rest:8181",
+            "credential": principal_credential,
+            "uri": "http://rest:8181/api/catalog",
             "s3.endpoint": "http://minio:9000",
             "s3.access-key-id": "admin",
             "s3.secret-access-key": "password",
+            "warehouse": "polaris",
+            "scope": "PRINCIPAL_ROLE:ALL"
         },
     ),
     'hive': load_catalog(

dev/spark-defaults.conf

@@ -18,9 +18,11 @@
 spark.sql.extensions                   org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions
 spark.sql.catalog.rest                 org.apache.iceberg.spark.SparkCatalog
 spark.sql.catalog.rest.type            rest
-spark.sql.catalog.rest.uri             http://rest:8181
+spark.sql.catalog.rest.uri             http://rest:8181/api/catalog
+spark.sql.catalog.rest.oauth2-server-uri             http://rest:8181/api/catalog/v1/oauth/tokens
 spark.sql.catalog.rest.io-impl         org.apache.iceberg.aws.s3.S3FileIO
-spark.sql.catalog.rest.warehouse       s3://warehouse/rest/
+spark.sql.catalog.rest.warehouse       polaris
+spark.sql.catalog.rest.scope           PRINCIPAL_ROLE:ALL
 spark.sql.catalog.rest.s3.endpoint     http://minio:9000
 spark.sql.catalog.hive                 org.apache.iceberg.spark.SparkCatalog
 spark.sql.catalog.hive.type            hive