From 2b4e08d8439386a16c123ff45cf4925ddc3bca2f Mon Sep 17 00:00:00 2001 From: rwxd Date: Mon, 6 Mar 2023 22:43:38 +0100 Subject: [PATCH] fix: init --- .github/ISSUE_TEMPLATE/bug_report_md | 45 ++ .github/ISSUE_TEMPLATE/config.yml | 2 + .github/ISSUE_TEMPLATE/feature_request.md | 37 ++ .github/dependabot.yml | 21 + .github/settings.yml | 5 + .github/workflows/release.yml | 66 +++ .github/workflows/semantic-release.yml | 11 + .gitignore | 155 +++++++ .pre-commit-config.yaml | 63 +++ .releaserc.yml | 10 + LICENSE | 21 + Makefile | 17 + README.md | 1 + docs/index.md | 37 ++ docs/models.md | 3 + docs/readwise-api.md | 5 + docs/readwise-reader-api.md | 5 + mkdocs.yaml | 88 ++++ readwise/__init__.py | 6 + readwise/api.py | 482 ++++++++++++++++++++++ readwise/models.py | 83 ++++ renovate.json | 4 + requirements-dev.txt | 1 + requirements-docs.txt | 3 + requirements.txt | 3 + setup.py | 39 ++ 26 files changed, 1213 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE/bug_report_md create mode 100644 .github/ISSUE_TEMPLATE/config.yml create mode 100644 .github/ISSUE_TEMPLATE/feature_request.md create mode 100644 .github/dependabot.yml create mode 100644 .github/settings.yml create mode 100644 .github/workflows/release.yml create mode 100644 .github/workflows/semantic-release.yml create mode 100644 .gitignore create mode 100644 .pre-commit-config.yaml create mode 100644 .releaserc.yml create mode 100644 LICENSE create mode 100644 Makefile create mode 100644 README.md create mode 100644 docs/index.md create mode 100644 docs/models.md create mode 100644 docs/readwise-api.md create mode 100644 docs/readwise-reader-api.md create mode 100644 mkdocs.yaml create mode 100644 readwise/__init__.py create mode 100644 readwise/api.py create mode 100644 readwise/models.py create mode 100644 renovate.json create mode 100644 requirements-dev.txt create mode 100644 requirements-docs.txt create mode 100644 requirements.txt create mode 100644 setup.py diff --git a/.github/ISSUE_TEMPLATE/bug_report_md b/.github/ISSUE_TEMPLATE/bug_report_md new file mode 100644 index 0000000..a786dae --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report_md @@ -0,0 +1,45 @@ +--- +name: "\U0001F41B Bug Report" +about: "If something isn't working as expected \U0001F914." +title: '' +labels: bug + +--- + + + + +### Used Release + + +### Debug Output + + +### Panic Output + + +### Expected Behavior +What should have happened? + +### Actual Behavior +What actually happened? + +### Steps to Reproduce + + +### Important Factoids + + +### References + + +### Community Note + +* Please vote on this issue by adding a 👍 [reaction](https://blog.github.com/2016-03-10-add-reactions-to-pull-requests-issues-and-comments/) to the original issue to help the community and maintainers prioritize this request +* If you are interested in working on this issue or have submitted a pull request, please leave a comment diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 0000000..bd9dfe4 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,2 @@ +--- +blank_issues_enabled: false diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 0000000..0ba88c2 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,37 @@ +--- +name: "\U0001F680 Feature Request" +about: "I have a suggestion (and might want to implement myself \U0001F642)!" +title: '' +labels: enhancement + +--- + + +### Description + + + +### Potential Configuration + + + +```hcl +# Copy-paste your configurations here. +``` + +### References + + + + + +### Community Note + +* Please vote on this issue by adding a 👍 [reaction](https://blog.github.com/2016-03-10-add-reactions-to-pull-requests-issues-and-comments/) to the original issue to help the community and maintainers prioritize this request +* If you are interested in working on this issue or have submitted a pull request, please leave a comment + + diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..f9c88ee --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,21 @@ +version: 2 +updates: + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "daily" + + - package-ecosystem: "pip" + directory: "/" + schedule: + interval: "daily" + + - package-ecosystem: "gitsubmodule" + directory: "/" + schedule: + interval: "daily" + + - package-ecosystem: "docker" + directory: "/" + schedule: + interval: "daily" diff --git a/.github/settings.yml b/.github/settings.yml new file mode 100644 index 0000000..9425d73 --- /dev/null +++ b/.github/settings.yml @@ -0,0 +1,5 @@ +repository: + name: "pyreadwise" + description: "Python Module to use the Readwise API" + homepage: "https://rwxd.github.io/pyreadwise/" + topics: "python, readwise" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..9e687a2 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,66 @@ +--- +on: + push: + branches: + - "*" + tags: + - "v*.*.*" + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: '3.10' + + - name: Debug + run: pwd && ls -la + + - name: Setup + run: make setup + + - name: Run pre-commit + run: pre-commit run --show-diff-on-failure --all-files + env: + SKIP: "no-commit-to-branch" + + pypi: + runs-on: ubuntu-latest + needs: + - test + if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags') + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: '3.10' + + - name: Install dependencies + run: pip install wheel + + - name: Build package + run: python setup.py sdist bdist_wheel + + - name: Publish package + uses: pypa/gh-action-pypi-publish@release/v1 + with: + password: ${{ secrets.PYPI_API_TOKEN }} + + docs: + runs-on: ubuntu-latest + needs: + - test + if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags') + steps: + - uses: actions/checkout@v3 + + - name: build docs + run: make build-docs + + - name: Deploy 🚀 + uses: JamesIves/github-pages-deploy-action@v4.3.4 + with: + BRANCH: pages + FOLDER: ./site diff --git a/.github/workflows/semantic-release.yml b/.github/workflows/semantic-release.yml new file mode 100644 index 0000000..b0ed427 --- /dev/null +++ b/.github/workflows/semantic-release.yml @@ -0,0 +1,11 @@ +--- +on: + push: + branches: + - main + +jobs: + semantic-release: + uses: rwxd/gh-templates/.github/workflows/common-semantic-release.yml@main + secrets: + token: ${{ secrets.GH_TOKEN }} diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..3adbb40 --- /dev/null +++ b/.gitignore @@ -0,0 +1,155 @@ +test.py +site/ + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintainted in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..359a575 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,63 @@ +--- +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: "v4.2.0" + hooks: + - id: check-yaml + - id: check-json + - id: end-of-file-fixer + - id: trailing-whitespace + - id: check-added-large-files + - id: check-symlinks + - id: no-commit-to-branch + - id: trailing-whitespace + - id: debug-statements + - id: double-quote-string-fixer + - id: requirements-txt-fixer + + - repo: https://github.com/psf/black + rev: "22.3.0" + hooks: + - id: black + args: + - "--skip-string-normalization" + + - repo: https://github.com/asottile/pyupgrade + rev: "v2.31.0" + hooks: + - id: pyupgrade + + - repo: https://github.com/pre-commit/mirrors-mypy + rev: "v0.991" + hooks: + - id: mypy + args: + - "--ignore-missing-imports" + additional_dependencies: + - types-requests + - types-pyyaml + + - repo: https://github.com/PyCQA/flake8 + rev: 4.0.1 + hooks: + - id: flake8 + args: + - "--max-line-length=100" + + - repo: https://github.com/pycqa/isort + rev: 5.10.1 + hooks: + - id: isort + name: isort (python) + args: + - "--profile" + - "black" + - "--filter-files" + + - repo: https://github.com/PyCQA/autoflake + rev: v1.5.3 + hooks: + - id: autoflake + args: + - "--remove-all-unused-imports" + - "--ignore-init-module-imports" diff --git a/.releaserc.yml b/.releaserc.yml new file mode 100644 index 0000000..04d9f38 --- /dev/null +++ b/.releaserc.yml @@ -0,0 +1,10 @@ +plugins: + - "@semantic-release/commit-analyzer" + - "@semantic-release/release-notes-generator" + - "@semantic-release/github" + +branches: + - "main" + - "+([0-9])?(.{+([0-9]),x}).x" + - name: "alpha" + prerelease: "alpha" diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..9301b92 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2023, rwxd + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..5fbb50d --- /dev/null +++ b/Makefile @@ -0,0 +1,17 @@ +PROJECT_NAME := "wallabag2readwise" + +help: + @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' + +setup: ## Setup required things + python3 -m pip install -U -r requirements.txt + python3 -m pip install -U -r requirements-dev.txt + python3 -m pip install -U -r requirements-docs.txt + pre-commit install + pre-commit install-hooks + +serve-docs: ## Serve the documentation locally + mkdocs serve + +build-docs: ## Build the documentation + mkdocs build diff --git a/README.md b/README.md new file mode 100644 index 0000000..1869091 --- /dev/null +++ b/README.md @@ -0,0 +1 @@ +# Python Readwise Module diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 0000000..f4a984a --- /dev/null +++ b/docs/index.md @@ -0,0 +1,37 @@ +# Python Module to use the Readwise API + +## Installation + +```bash +pip install -U readwise +``` + +## Quickstart + +### Readwise API + +```python +from readwise import ReadwiseClient + +client = ReadwiseClient('token') + +books = client.get_books(category='articles') + +for book in books: + highlights = client.get_book_highlights(book.id) + if len(highlights) > 0: + print(book.title) + for highlight in highlights: + print(highlight.text) +``` + +### Readwise Readwise API + +```python +from readwise import ReadwiseReaderClient + +client = ReadwiseReaderClient('token') + +response = client.create_document('https://www.example.com') +response.raise_for_status() +``` diff --git a/docs/models.md b/docs/models.md new file mode 100644 index 0000000..bd7a7d4 --- /dev/null +++ b/docs/models.md @@ -0,0 +1,3 @@ +# Models + +::: readwise.models diff --git a/docs/readwise-api.md b/docs/readwise-api.md new file mode 100644 index 0000000..73ae8c4 --- /dev/null +++ b/docs/readwise-api.md @@ -0,0 +1,5 @@ +# Readwise API + +::: readwise.api.ReadwiseClient + options: + show_if_no_docstring: true diff --git a/docs/readwise-reader-api.md b/docs/readwise-reader-api.md new file mode 100644 index 0000000..55e3c7b --- /dev/null +++ b/docs/readwise-reader-api.md @@ -0,0 +1,5 @@ +# Readwise Reader API + +::: readwise.api.ReadwiseReaderClient + options: + show_if_no_docstring: true diff --git a/mkdocs.yaml b/mkdocs.yaml new file mode 100644 index 0000000..c09a978 --- /dev/null +++ b/mkdocs.yaml @@ -0,0 +1,88 @@ +--- +site_name: pyreadwise +site_description: Python module to use the readwise api +site_author: rwxd +site_url: https://rwxd.github.io/pyreadwise +dev_addr: 0.0.0.0:8000 + +repo_name: "pyreadwise" +repo_url: "https://github.com/rwxd/pyreadwise" + +docs_dir: "./docs" +edit_uri: "edit/source/docs/content/" + +# https://squidfunk.github.io/mkdocs-material/ +theme: + name: "material" + palette: + - scheme: slate + primary: teal + accent: teal + toggle: + icon: material/toggle-switch + name: Switch to light mode + - scheme: default + primary: teal + accent: teal + toggle: + icon: material/toggle-switch-off-outline + name: Switch to dark mode + icon: + logo: material/code-braces-box + # logo: material/pine-tree + # favicon: assets/page/favicon-32x32.png + language: en + include_sidebar: true + features: + - content.code.annotate + feature: + tabs: false + i18n: + prev: "Previous" + next: "Next" + font: + text: Inter + code: Fira Code + +copyright: "Copyright © 2023 rwxd" + +plugins: + - search + - mkdocstrings: + handlers: + python: + import: + - https://docs.python-requests.org/en/master/objects.inv + options: + docstring_style: google + docstring_section_style: "table" + merge_init_into_class: True + line_length: 100 + members_order: 'source' + + show_signature_annotations: True + separate_signature: True + +markdown_extensions: + - pymdownx.highlight: + linenums: true + linenums_style: pymdownx-inline + guess_lang: true + - pymdownx.superfences + - pymdownx.inlinehilite + - pymdownx.keys + - pymdownx.snippets + - toc: + permalink: "⚑" + +nav: + - "Home": 'index.md' + - "Code Reference": + - "Readwise API": readwise-api.md + - "Readwise Reader API": readwise-reader-api.md + - "Models": models.md + +extra: + social: + - icon: fontawesome/brands/github + link: https://github.com/rwxd diff --git a/readwise/__init__.py b/readwise/__init__.py new file mode 100644 index 0000000..f278415 --- /dev/null +++ b/readwise/__init__.py @@ -0,0 +1,6 @@ +from readwise.api import ReadwiseClient, ReadwiseReaderClient + +__all__ = [ + 'ReadwiseClient', + 'ReadwiseReaderClient', +] diff --git a/readwise/api.py b/readwise/api.py new file mode 100644 index 0000000..31e0b89 --- /dev/null +++ b/readwise/api.py @@ -0,0 +1,482 @@ +import logging +from datetime import datetime +from time import sleep +from typing import Any, Generator, Literal + +import requests +from backoff import expo, on_exception +from ratelimit import RateLimitException, limits, sleep_and_retry + +from readwise.models import ReadwiseBook, ReadwiseHighlight, ReadwiseTag + + +class ReadwiseRateLimitException(Exception): + '''Raised when the Readwise API rate limit is exceeded.''' + + pass + + +class ReadwiseClient: + def __init__( + self, + token: str, + ): + ''' + Initialize a ReadwiseClient. + + Documentation for the Readwise API can be found here: + https://readwise.io/api_deets + + Args: + token: Readwise API token + ''' + self.token = token + self.url = 'https://readwise.io/api/v2' + + @property + def _session(self) -> requests.Session: + ''' + Return a requests.Session object with the Readwise API token set as an + Authorization header. + ''' + session = requests.Session() + session.headers.update( + { + 'Accept': 'application/json', + 'Authorization': f'Token {self.token}', + } + ) + return session + + @on_exception(expo, RateLimitException, max_tries=8) + @sleep_and_retry + @limits(calls=240, period=60) + def _request( + self, method: str, endpoint: str, params: dict = {}, data: dict = {} + ) -> requests.Response: + ''' + Make a request to the Readwise API. + + The Readwise API has a rate limit of 240 requests per minute. This + method will raise a ReadwiseRateLimitException if the rate limit is + exceeded. + + The Exception will be raised after 8 retries with exponential backoff. + + Args: + method: HTTP method + endpoint: API endpoint + params: Query parameters + data: Request body + + Returns: + requests.Response + ''' + url = self.url + endpoint + logging.debug(f'Calling "{method}" on "{url}" with params: {params}') + response = self._session.request(method, url, params=params, json=data) + while response.status_code == 429: + seconds = int(response.headers['Retry-After']) + logging.warning(f'Rate limited by Readwise, retrying in {seconds} seconds') + sleep(seconds) + response = self._session.request(method, url, params=params, data=data) + response.raise_for_status() + return response + + def get(self, endpoint: str, params: dict = {}) -> requests.Response: + ''' + Make a GET request to the Readwise API. + + Examples: + >>> client.get('/highlights/') + + Args: + endpoint: API endpoint + params: Query parameters + + Returns: + requests.Response + ''' + logging.debug(f'Getting "{endpoint}" with params: {params}') + return self._request('GET', endpoint, params=params) + + @on_exception(expo, RateLimitException, max_tries=8) + @sleep_and_retry + @limits(calls=20, period=60) + def get_with_limit_20(self, endpoint: str, params: dict = {}) -> requests.Response: + ''' + Get a response from the Readwise API with a rate limit of 20 requests + per minute. + + The rate limit of 20 requests per minute needs to be used at the + endpoints /highlights/ and /books/ because they return a lot of data. + + Args: + endpoint: API endpoint + params: Query parameters + Returns: + requests.Response + ''' + return self.get(endpoint, params) + + def post(self, endpoint: str, data: dict = {}) -> requests.Response: + ''' + Make a POST request to the Readwise API. + + Examples: + >>> client.post('/highlights/', {'highlights': [{'text': 'foo'}]}) + + Args: + endpoint: API endpoint + data: Request body + + Returns: + requests.Response + ''' + url = self.url + endpoint + logging.debug(f'Posting "{url}" with data: {data}') + response = self._request('POST', endpoint, data=data) + response.raise_for_status() + return response + + def delete(self, endpoint: str) -> requests.Response: + ''' + Make a DELETE request to the Readwise API. + + Examples: + >>> client.delete('/highlights/1234') + + Args: + endpoint: API endpoint + + Returns: + requests.Response + ''' + logging.debug(f'Deleting "{endpoint}"') + return self._request('DELETE', endpoint) + + def get_books( + self, category: Literal['articles', 'books', 'tweets', 'podcasts'] + ) -> Generator[ReadwiseBook, None, None]: + ''' + Get all Readwise books. + + Args: + category: Book category + + Returns: + A generator of ReadwiseBook objects + ''' + page = 1 + page_size = 1000 + while True: + data = self.get_with_limit_20( + '/books', + {'page': page, 'page_size': page_size, 'category': category}, + ).json() + + for book in data['results']: + yield ReadwiseBook( + id=book['id'], + title=book['title'], + author=book['author'], + category=book['category'], + source=book['source'], + num_highlights=book['num_highlights'], + last_highlight_at=datetime.fromisoformat(book['last_highlight_at']), + updated=datetime.fromisoformat(book['updated']), + cover_image_url=book['cover_image_url'], + highlights_url=book['highlights_url'], + source_url=book['source_url'], + asin=book['asin'], + tags=[ + ReadwiseTag(id=tag['id'], name=tag['name']) + for tag in book['tags'] + ], + document_note=book['document_note'], + ) + + if not data['next']: + break + page += 1 + + def get_book_highlights( + self, book_id: str + ) -> Generator[ReadwiseHighlight, None, None]: + ''' + Get all highlights for a Readwise book. + + Args: + book_id: Readwise book ID + + Returns: + A generator of ReadwiseHighlight objects + ''' + page = 1 + page_size = 1000 + while True: + data = self.get_with_limit_20( + '/highlights', + {'page': page, 'page_size': page_size, 'book_id': book_id}, + ).json() + for highlight in data['results']: + yield ReadwiseHighlight( + id=highlight['id'], + text=highlight['text'], + note=highlight['note'], + location=highlight['location'], + location_type=highlight['location_type'], + url=highlight['url'], + color=highlight['color'], + updated=datetime.fromisoformat(highlight['updated']), + book_id=highlight['book_id'], + tags=[ + ReadwiseTag(id=tag['id'], name=tag['name']) + for tag in highlight['tags'] + ], + ) + + if not data['next']: + break + page += 1 + + def create_highlight( + self, + text: str, + title: str, + author: str | None = None, + highlighted_at: datetime | None = None, + source_url: str | None = None, + category: str = 'articles', + note: str | None = None, + ): + ''' + Create a Readwise highlight. + + Args: + text: Highlight text + title: Book title + author: Book author + highlighted_at: Date and time the highlight was created + source_url: URL of the book + category: Book category + note: Highlight note + ''' + payload = {'text': text, 'title': title, 'category': category} + if author: + payload['author'] = author + if highlighted_at: + payload['highlighted_at'] = highlighted_at.isoformat() + if source_url: + payload['source_url'] = source_url + if note: + payload['note'] = note + + self.post('/highlights/', {'highlights': [payload]}) + + def get_book_tags(self, book_id: str) -> Generator[ReadwiseTag, None, None]: + ''' + Get all tags for a Readwise book. + + Args: + book_id: Readwise book ID + + Returns: + A generator of ReadwiseTag objects + ''' + page = 1 + page_size = 1000 + data = self.get( + f'/books/{book_id}/tags', + {'page': page, 'page_size': page_size, 'book_id': book_id}, + ).json() + + for tag in data: + yield ReadwiseTag(tag['id'], tag['name']) + + def add_tag(self, book_id: str, tag: str): + ''' + Add a tag to a Readwise book. + + Args: + book_id: Readwise book ID + tag: Tag name + + Returns: + requests.Response + ''' + logging.debug(f'Adding tag "{tag}" to book "{book_id}"') + payload = {'name': tag} + self.post(f'/books/{book_id}/tags/', payload) + + def delete_tag(self, book_id: str, tag_id: str): + ''' + Delete a tag from a Readwise book. + + Args: + book_id: Readwise book ID + + Returns: + requests.Response + ''' + logging.debug(f'Deleting tag "{tag_id}"') + self.delete(f'/books/{book_id}/tags/{tag_id}') + + +class ReadwiseReaderClient: + def __init__( + self, + token: str, + ): + ''' + Readwise Reader Connector. + + Documentation for the Readwise Reader API can be found here: + https://readwise.io/reader_api + + Args: + token: Readwise Reader Connector token + ''' + self.token = token + self.url = 'https://readwise.io/api/v3' + + @property + def _session(self) -> requests.Session: + ''' + Session object for making requests. + The headers are set to include the token. + ''' + session = requests.Session() + session.headers.update( + { + 'Accept': 'application/json', + 'Authorization': f'Token {self.token}', + } + ) + return session + + @on_exception(expo, RateLimitException, max_tries=8) + @sleep_and_retry + @limits(calls=20, period=60) + def _request( + self, method: str, endpoint: str, params: dict = {}, data: dict = {} + ) -> requests.Response: + ''' + Make a request to the Readwise Reader API. + The request is rate limited to 20 calls per minute. + + Args: + method: HTTP method + endpoint: API endpoints + params: Query parameters + data: Request body + + Returns: + requests.Response + ''' + url = self.url + endpoint + logging.debug(f'Calling "{method}" on "{url}" with params: {params}') + response = self._session.request(method, url, params=params, json=data) + while response.status_code == 429: + seconds = int(response.headers['Retry-After']) + logging.warning(f'Rate limited by Readwise, retrying in {seconds} seconds') + sleep(seconds) + response = self._session.request(method, url, params=params, data=data) + response.raise_for_status() + return response + + def get(self, endpoint: str, params: dict = {}) -> requests.Response: + ''' + Make a GET request to the Readwise Reader API. + + Args: + endpoint: API endpoints + params: Query parameters + + Returns: + requests.Response + ''' + logging.debug(f'Getting "{endpoint}" with params: {params}') + return self._request('GET', endpoint, params=params) + + def post(self, endpoint: str, data: dict = {}) -> requests.Response: + ''' + Make a POST request to the Readwise Reader API. + + Args: + endpoint: API endpoints + data: Request body + + Returns: + requests.Response + ''' + url = self.url + endpoint + logging.debug(f'Posting "{url}" with data: {data}') + response = self._request('POST', endpoint, data=data) + response.raise_for_status() + return response + + def create_document( + self, + url: str, + html: str | None = None, + should_clean_html: bool | None = None, + title: str | None = None, + author: str | None = None, + summary: str | None = None, + published_at: datetime | None = None, + image_url: str | None = None, + location: Literal['new', 'later', 'archive', 'feed'] = 'new', + saved_using: str | None = None, + tags: list[str] = [], + ) -> requests.Response: + ''' + Create a document in Readwise Reader. + + Args: + url: Document URL + html: Document HTML + should_clean_html: Whether to clean the HTML + title: Document title + author: Document author + summary: Document summary + published_at: Date and time the document was published + image_url: An image URL to use as cover image + location: Document location + saved_using: How the document was saved + tags: List of tags + + Returns: + requests.Response + ''' + data: dict[str, Any] = { + 'url': url, + 'tags': tags, + 'location': location, + } + + if html: + data['html'] = html + + if should_clean_html is not None: + data['should_clean_html'] = should_clean_html + + if title: + data['title'] = title + + if author: + data['author'] = author + + if summary: + data['summary'] = summary + + if published_at: + data['published_at'] = published_at.isoformat() + + if image_url: + data['image_url'] = image_url + + if saved_using: + data['saved_using'] = saved_using + + return self.post('/save/', data) diff --git a/readwise/models.py b/readwise/models.py new file mode 100644 index 0000000..d2b254f --- /dev/null +++ b/readwise/models.py @@ -0,0 +1,83 @@ +from dataclasses import dataclass +from datetime import datetime + + +@dataclass +class ReadwiseTag: + '''Represents a Readwise tag. + + Attributes: + id: The tag's ID. + name: The tag's name. + ''' + + id: str + name: str + + +@dataclass +class ReadwiseBook: + ''' + Represents a Readwise book. + + Attributes: + id: The book's ID. + title: The book's title. + author: The book's author. + category: The book's category. + source: The book's source. + num_highlights: The number of highlights for the book. + last_highlight_at: The date and time of the last highlight for the book. + updated: The date and time the book was last updated. + cover_image_url: The URL of the book's cover image. + highlights_url: The URL of the book's highlights. + source_url: The URL of the book's source. + asin: The book's ASIN. + tags: The book's tags. + document_note: The book's document note. + ''' + + id: str + title: str + author: str + category: str + source: str + num_highlights: int + last_highlight_at: datetime + updated: datetime + cover_image_url: str + highlights_url: str + source_url: str + asin: str + tags: list[ReadwiseTag] + document_note: str + + +@dataclass +class ReadwiseHighlight: + ''' + Represents a Readwise highlight. + + Attributes: + id: The highlight's ID. + text: The highlight's text. + note: The highlight's note. + location: The highlight's location. + location_type: The highlight's location type. + url: The highlight's URL. + color: The highlight's color. + updated: The date and time the highlight was last updated. + book_id: The ID of the book the highlight is from. + tags: The highlight's tags. + ''' + + id: str + text: str + note: str + location: int + location_type: str + url: str | None + color: str + updated: datetime + book_id: str + tags: list[ReadwiseTag] diff --git a/renovate.json b/renovate.json new file mode 100644 index 0000000..c17cafe --- /dev/null +++ b/renovate.json @@ -0,0 +1,4 @@ +{ + "$schema": "https://docs.renovatebot.com/renovate-schema.json", + "extends": ["github>rwxd/renovate-config"] +} diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..561cab7 --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1 @@ +pre-commit==2.20.0 diff --git a/requirements-docs.txt b/requirements-docs.txt new file mode 100644 index 0000000..de6eaed --- /dev/null +++ b/requirements-docs.txt @@ -0,0 +1,3 @@ +mkdocs==1.4.2 +mkdocs-material==9.0.0 +mkdocstrings[python]==0.20.0 diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..083c04c --- /dev/null +++ b/requirements.txt @@ -0,0 +1,3 @@ +backoff==2.2.1 +ratelimit==2.2.1 +requests==2.28.1 diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..bfb68ab --- /dev/null +++ b/setup.py @@ -0,0 +1,39 @@ +from os import path +from subprocess import SubprocessError, check_output + +from setuptools import setup + +with open('./requirements.txt') as f: + required = f.read().splitlines() + +this_directory = path.abspath(path.dirname(__file__)) +with open(path.join(this_directory, 'README.md'), encoding='utf-8') as f: + long_description = f.read() + +version = '1.0.0' + +try: + version = ( + check_output(['git', 'describe', '--tags']).strip().decode().replace('v', '') + ) +except SubprocessError as e: + print(e) + +setup( + name='readwise', + version=version, + description='Readwise api client', + long_description=long_description, + long_description_content_type='text/markdown', + author='rwxd', + author_email='rwxd@pm.me', + url='https://github.com/rwxd/pyreadwise', + license='MIT', + packages=['readwise'], + install_requires=required, + classifiers=[ + 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.10', + 'Programming Language :: Python :: 3.11', + ], +)