Fixes to get the Zoom tap working

.gitignore

@@ -119,3 +119,5 @@ rsa-key
 tags
 singer-check-tap-data
 state.json
+catalog.json
+

tap_zoom/client.py

@@ -28,16 +28,24 @@ def __init__(self, config, config_path):
         self.__session = requests.Session()
         self.__config_path = config_path
         self.__access_token = None
+        self.__refresh_token = None
         self.__use_jwt = False
 
         jwt = config.get('jwt')
+        refresh_token = config.get('refresh_token')
         if jwt:
             self.__access_token = jwt
             self.__use_jwt = True
         else:
             self.__client_id = config.get('client_id')
             self.__client_secret = config.get('client_secret')
-            self.__refresh_token = config.get('refresh_token')
+            if refresh_token:
+                self.__refresh_token = config.get('refresh_token')
+            else:
+                # For server-to-server oauth apps, there are no refresh
+                # tokens. We use the persistent access token.
+                # https://marketplace.zoom.us/docs/guides/build/server-to-server-oauth-app/
+                self.__access_token = config.get('access_token')
 
     def __enter__(self):
         return self
@@ -54,7 +62,6 @@ def refresh_access_token(self):
                 'refresh_token': self.__refresh_token,
                 'grant_type': 'refresh_token'
             })
-
         self.__access_token = data['access_token']
         self.__refresh_token = data['refresh_token']
 
@@ -81,8 +88,8 @@ def request(self,
                 ignore_zoom_error_codes=[],
                 ignore_http_error_codes=[],
                 **kwargs):
-        if url is None and \
-            self.__use_jwt == False and \
+        uses_refresh_token = self.__use_jwt == False and self.__refresh_token is not None
+        if uses_refresh_token and url is None and \
             (self.__access_token is None or \
              self.__expires_at <= datetime.utcnow()):
             self.refresh_access_token()

tap_zoom/discover.py

@@ -5,6 +5,7 @@
 
 from tap_zoom.endpoints import ENDPOINTS_CONFIG
 
+
 SCHEMAS = {}
 FIELD_METADATA = {}
 
@@ -44,7 +45,14 @@ def get_schemas():
 
         pk = get_pk(stream_name)
 
-        metadata = []
+        metadata = [{
+            'metadata': {
+                'selected': False,
+                'inclusion': 'available',
+                'table-key-properties': pk,
+            },
+            'breadcrumb': []
+        }]
         for prop, json_schema in schema['properties'].items():
             if prop in pk:
                 inclusion = 'automatic'

tap_zoom/endpoints.py

@@ -80,7 +80,7 @@
                         'path': 'webinars/{webinar_id}',
                         'pk': ['uuid'],
                         'provides': {
-                            'webinar_uuid': 'uuid'
+                            'webinar_uuid': 'id'
                         },
                         'children': {
                             'webinar_absentees': {

tap_zoom/sync.py

@@ -22,6 +22,7 @@ def write_schema(stream):
     schema = stream.schema.to_dict()
     singer.write_schema(stream.tap_stream_id, schema, stream.key_properties)
 
+
 def sync_endpoint(client,
                   catalog,
                   state,
@@ -41,11 +42,11 @@ def sync_endpoint(client,
     path = endpoint['path'].format(**key_bag)
 
     page_size = 1000
-    page_number = 1
+    next_page_token = ''
     while True:
         params = {
             'page_size': page_size,
-            'page_number': page_number
+            'next_page_token': next_page_token
         }
 
         data = client.get(path,
@@ -88,10 +89,10 @@ def sync_endpoint(client,
                                               child_endpoint,
                                               child_key_bag)
 
-        if endpoint.get('paginate', True) and page_number < data.get('page_count', 1):
+        if endpoint.get('paginate', True) and data.get('next_page_token', ''):
             # each endpoint has a different max page size, the server will send the one that is forced
             page_size = data['page_size']
-            page_number += 1
+            next_page_token = data['next_page_token']
         else:
             break
 
@@ -110,6 +111,7 @@ def get_required_streams(endpoints, selected_stream_names):
             required_streams.append(name)
             if child_required_streams:
                 required_streams += child_required_streams
+
     return required_streams
 
 def sync(client, catalog, state):