Merge pull request #18 from refly-ai/feat/client-upload-and-read

Feat/client upload and read

aero/.gitignore

@@ -0,0 +1,181 @@
+# Created by https://www.toptal.com/developers/gitignore/api/python
+# Edit at https://www.toptal.com/developers/gitignore?templates=python
+
+### Python ###
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/#use-with-ide
+.pdm.toml
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/
+
+### Python Patch ###
+# Poetry local configuration file - https://python-poetry.org/docs/configuration/#local-configuration
+poetry.toml
+
+# ruff
+.ruff_cache/
+
+# LSP config files
+pyrightconfig.json
+
+# End of https://www.toptal.com/developers/gitignore/api/python
+唤起 Refly
+⌘B
+关闭
+Refly
+Refly

aero/aero_pb2.pyi

@@ -0,0 +1,38 @@
+from google.protobuf.internal import containers as _containers
+from google.protobuf.internal import enum_type_wrapper as _enum_type_wrapper
+from google.protobuf import descriptor as _descriptor
+from google.protobuf import message as _message
+from typing import ClassVar as _ClassVar, Iterable as _Iterable, Mapping as _Mapping, Optional as _Optional, Union as _Union
+
+DESCRIPTOR: _descriptor.FileDescriptor
+
+class SelectorType(int, metaclass=_enum_type_wrapper.EnumTypeWrapper):
+    __slots__ = ()
+    XPath: _ClassVar[SelectorType]
+XPath: SelectorType
+
+class ParseHTMLRequest(_message.Message):
+    __slots__ = ("html", "selector_type", "selectors")
+    HTML_FIELD_NUMBER: _ClassVar[int]
+    SELECTOR_TYPE_FIELD_NUMBER: _ClassVar[int]
+    SELECTORS_FIELD_NUMBER: _ClassVar[int]
+    html: str
+    selector_type: SelectorType
+    selectors: _containers.RepeatedScalarFieldContainer[str]
+    def __init__(self, html: _Optional[str] = ..., selector_type: _Optional[_Union[SelectorType, str]] = ..., selectors: _Optional[_Iterable[str]] = ...) -> None: ...
+
+class HTMLFragment(_message.Message):
+    __slots__ = ("selector_type", "selector", "content")
+    SELECTOR_TYPE_FIELD_NUMBER: _ClassVar[int]
+    SELECTOR_FIELD_NUMBER: _ClassVar[int]
+    CONTENT_FIELD_NUMBER: _ClassVar[int]
+    selector_type: SelectorType
+    selector: str
+    content: str
+    def __init__(self, selector_type: _Optional[_Union[SelectorType, str]] = ..., selector: _Optional[str] = ..., content: _Optional[str] = ...) -> None: ...
+
+class ParseHTMLResult(_message.Message):
+    __slots__ = ("fragments",)
+    FRAGMENTS_FIELD_NUMBER: _ClassVar[int]
+    fragments: _containers.RepeatedCompositeFieldContainer[HTMLFragment]
+    def __init__(self, fragments: _Optional[_Iterable[_Union[HTMLFragment, _Mapping]]] = ...) -> None: ...

aero/aero_pb2_grpc.py

@@ -0,0 +1,70 @@
+# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT!
+"""Client and server classes corresponding to protobuf-defined services."""
+import grpc
+
+import aero_pb2 as aero__pb2
+
+
+class AeroStub(object):
+    """The aero service definition.
+    """
+
+    def __init__(self, channel):
+        """Constructor.
+
+        Args:
+            channel: A grpc.Channel.
+        """
+        self.ParseHTML = channel.unary_unary(
+                '/aero.Aero/ParseHTML',
+                request_serializer=aero__pb2.ParseHTMLRequest.SerializeToString,
+                response_deserializer=aero__pb2.ParseHTMLResult.FromString,
+                )
+
+
+class AeroServicer(object):
+    """The aero service definition.
+    """
+
+    def ParseHTML(self, request, context):
+        """Parse HTML
+        """
+        context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+        context.set_details('Method not implemented!')
+        raise NotImplementedError('Method not implemented!')
+
+
+def add_AeroServicer_to_server(servicer, server):
+    rpc_method_handlers = {
+            'ParseHTML': grpc.unary_unary_rpc_method_handler(
+                    servicer.ParseHTML,
+                    request_deserializer=aero__pb2.ParseHTMLRequest.FromString,
+                    response_serializer=aero__pb2.ParseHTMLResult.SerializeToString,
+            ),
+    }
+    generic_handler = grpc.method_handlers_generic_handler(
+            'aero.Aero', rpc_method_handlers)
+    server.add_generic_rpc_handlers((generic_handler,))
+
+
+ # This class is part of an EXPERIMENTAL API.
+class Aero(object):
+    """The aero service definition.
+    """
+
+    @staticmethod
+    def ParseHTML(request,
+            target,
+            options=(),
+            channel_credentials=None,
+            call_credentials=None,
+            insecure=False,
+            compression=None,
+            wait_for_ready=None,
+            timeout=None,
+            metadata=None):
+        return grpc.experimental.unary_unary(request, target, '/aero.Aero/ParseHTML',
+            aero__pb2.ParseHTMLRequest.SerializeToString,
+            aero__pb2.ParseHTMLResult.FromString,
+            options, channel_credentials,
+            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)

aero/main.py

@@ -0,0 +1,59 @@
+import asyncio
+import logging
+from lxml import etree
+
+import grpc
+import aero_pb2
+import aero_pb2_grpc
+
+# Coroutines to be invoked when the event loop is shutting down.
+_cleanup_coroutines = []
+
+
+class Greeter(aero_pb2_grpc.AeroServicer):
+    async def ParseHTML(
+        self,
+        request: aero_pb2.ParseHTMLRequest,
+        context: grpc.aio.ServicerContext,
+    ) -> aero_pb2.ParseHTMLResult:
+        logging.info("Received request, parsing HTML")
+        tree = etree.HTML(request.html)
+        fragments = [
+            aero_pb2.HTMLFragment(
+                selector=selector,
+                selector_type=request.selector_type,
+                content=tree.xpath(selector)[0].text if tree.xpath(selector) else "", 
+            )
+            for selector in request.selectors
+        ]
+
+        return aero_pb2.ParseHTMLResult(fragments=fragments)
+
+
+async def serve() -> None:
+    server = grpc.aio.server()
+    aero_pb2_grpc.add_AeroServicer_to_server(Greeter(), server)
+    listen_addr = "[::]:50051"
+    server.add_insecure_port(listen_addr)
+    logging.info("Starting server on %s", listen_addr)
+    await server.start()
+
+    async def server_graceful_shutdown():
+        logging.info("Starting graceful shutdown...")
+        # Shuts down the server with 5 seconds of grace period. During the
+        # grace period, the server won't accept new connections and allow
+        # existing RPCs to continue within the grace period.
+        await server.stop(5)
+
+    _cleanup_coroutines.append(server_graceful_shutdown())
+    await server.wait_for_termination()
+
+
+if __name__ == "__main__":
+    logging.basicConfig(level=logging.INFO)
+    loop = asyncio.get_event_loop()
+    try:
+        loop.run_until_complete(serve())
+    finally:
+        loop.run_until_complete(*_cleanup_coroutines)
+        loop.close()

aero/protogen.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+python -m grpc_tools.protoc -I ../protos --python_out=. --pyi_out=. --grpc_python_out=. ../protos/aero.proto

aero/requirements.txt

@@ -0,0 +1,5 @@
+grpcio==1.62.1
+grpcio-tools==1.62.1
+lxml==5.2.1
+protobuf==4.25.3
+setuptools==69.5.1