Hackathon_PEC (1).ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "source": [
        "#!pip install llama-index-embeddings-huggingface\n",
        "#!pip install torch sentence-transformers\n",
        "#!pip install llama_index\n",
        "#!pip install tensorflow-io\n",
        "#!pip install llama-index-vector-stores-chroma"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "collapsed": true,
        "id": "QUxBCR5n7LtQ",
        "outputId": "4601ccda-df7e-4569-92fe-7059ca981d51"
      },
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: llama-index-embeddings-huggingface in /usr/local/lib/python3.10/dist-packages (0.2.2)\n",
            "Requirement already satisfied: huggingface-hub[inference]>=0.19.0 in /usr/local/lib/python3.10/dist-packages (from llama-index-embeddings-huggingface) (0.23.4)\n",
            "Requirement already satisfied: llama-index-core<0.11.0,>=0.10.1 in /usr/local/lib/python3.10/dist-packages (from llama-index-embeddings-huggingface) (0.10.55)\n",
            "Requirement already satisfied: sentence-transformers>=2.6.1 in /usr/local/lib/python3.10/dist-packages (from llama-index-embeddings-huggingface) (3.0.1)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from huggingface-hub[inference]>=0.19.0->llama-index-embeddings-huggingface) (3.15.4)\n",
            "Requirement already satisfied: fsspec>=2023.5.0 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub[inference]>=0.19.0->llama-index-embeddings-huggingface) (2023.6.0)\n",
            "Requirement already satisfied: packaging>=20.9 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub[inference]>=0.19.0->llama-index-embeddings-huggingface) (24.1)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub[inference]>=0.19.0->llama-index-embeddings-huggingface) (6.0.1)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from huggingface-hub[inference]>=0.19.0->llama-index-embeddings-huggingface) (2.31.0)\n",
            "Requirement already satisfied: tqdm>=4.42.1 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub[inference]>=0.19.0->llama-index-embeddings-huggingface) (4.66.4)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub[inference]>=0.19.0->llama-index-embeddings-huggingface) (4.12.2)\n",
            "Requirement already satisfied: aiohttp in /usr/local/lib/python3.10/dist-packages (from huggingface-hub[inference]>=0.19.0->llama-index-embeddings-huggingface) (3.9.5)\n",
            "Requirement already satisfied: minijinja>=1.0 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub[inference]>=0.19.0->llama-index-embeddings-huggingface) (2.0.1)\n",
            "Requirement already satisfied: SQLAlchemy[asyncio]>=1.4.49 in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-embeddings-huggingface) (2.0.31)\n",
            "Requirement already satisfied: dataclasses-json in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-embeddings-huggingface) (0.6.7)\n",
            "Requirement already satisfied: deprecated>=1.2.9.3 in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-embeddings-huggingface) (1.2.14)\n",
            "Requirement already satisfied: dirtyjson<2.0.0,>=1.0.8 in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-embeddings-huggingface) (1.0.8)\n",
            "Requirement already satisfied: httpx in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-embeddings-huggingface) (0.27.0)\n",
            "Requirement already satisfied: nest-asyncio<2.0.0,>=1.5.8 in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-embeddings-huggingface) (1.6.0)\n",
            "Requirement already satisfied: networkx>=3.0 in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-embeddings-huggingface) (3.3)\n",
            "Requirement already satisfied: nltk<4.0.0,>=3.8.1 in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-embeddings-huggingface) (3.8.1)\n",
            "Requirement already satisfied: numpy<2.0.0 in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-embeddings-huggingface) (1.25.2)\n",
            "Requirement already satisfied: openai>=1.1.0 in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-embeddings-huggingface) (1.35.14)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-embeddings-huggingface) (2.0.3)\n",
            "Requirement already satisfied: pillow>=9.0.0 in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-embeddings-huggingface) (9.4.0)\n",
            "Requirement already satisfied: tenacity!=8.4.0,<9.0.0,>=8.2.0 in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-embeddings-huggingface) (8.5.0)\n",
            "Requirement already satisfied: tiktoken>=0.3.3 in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-embeddings-huggingface) (0.7.0)\n",
            "Requirement already satisfied: typing-inspect>=0.8.0 in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-embeddings-huggingface) (0.9.0)\n",
            "Requirement already satisfied: wrapt in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-embeddings-huggingface) (1.14.1)\n",
            "Requirement already satisfied: transformers<5.0.0,>=4.34.0 in /usr/local/lib/python3.10/dist-packages (from sentence-transformers>=2.6.1->llama-index-embeddings-huggingface) (4.41.2)\n",
            "Requirement already satisfied: torch>=1.11.0 in /usr/local/lib/python3.10/dist-packages (from sentence-transformers>=2.6.1->llama-index-embeddings-huggingface) (2.3.0+cu121)\n",
            "Requirement already satisfied: scikit-learn in /usr/local/lib/python3.10/dist-packages (from sentence-transformers>=2.6.1->llama-index-embeddings-huggingface) (1.2.2)\n",
            "Requirement already satisfied: scipy in /usr/local/lib/python3.10/dist-packages (from sentence-transformers>=2.6.1->llama-index-embeddings-huggingface) (1.11.4)\n",
            "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp->huggingface-hub[inference]>=0.19.0->llama-index-embeddings-huggingface) (1.3.1)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->huggingface-hub[inference]>=0.19.0->llama-index-embeddings-huggingface) (23.2.0)\n",
            "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp->huggingface-hub[inference]>=0.19.0->llama-index-embeddings-huggingface) (1.4.1)\n",
            "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp->huggingface-hub[inference]>=0.19.0->llama-index-embeddings-huggingface) (6.0.5)\n",
            "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->huggingface-hub[inference]>=0.19.0->llama-index-embeddings-huggingface) (1.9.4)\n",
            "Requirement already satisfied: async-timeout<5.0,>=4.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->huggingface-hub[inference]>=0.19.0->llama-index-embeddings-huggingface) (4.0.3)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.10/dist-packages (from nltk<4.0.0,>=3.8.1->llama-index-core<0.11.0,>=0.10.1->llama-index-embeddings-huggingface) (8.1.7)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.10/dist-packages (from nltk<4.0.0,>=3.8.1->llama-index-core<0.11.0,>=0.10.1->llama-index-embeddings-huggingface) (1.4.2)\n",
            "Requirement already satisfied: regex>=2021.8.3 in /usr/local/lib/python3.10/dist-packages (from nltk<4.0.0,>=3.8.1->llama-index-core<0.11.0,>=0.10.1->llama-index-embeddings-huggingface) (2024.5.15)\n",
            "Requirement already satisfied: anyio<5,>=3.5.0 in /usr/local/lib/python3.10/dist-packages (from openai>=1.1.0->llama-index-core<0.11.0,>=0.10.1->llama-index-embeddings-huggingface) (3.7.1)\n",
            "Requirement already satisfied: distro<2,>=1.7.0 in /usr/lib/python3/dist-packages (from openai>=1.1.0->llama-index-core<0.11.0,>=0.10.1->llama-index-embeddings-huggingface) (1.7.0)\n",
            "Requirement already satisfied: pydantic<3,>=1.9.0 in /usr/local/lib/python3.10/dist-packages (from openai>=1.1.0->llama-index-core<0.11.0,>=0.10.1->llama-index-embeddings-huggingface) (2.8.2)\n",
            "Requirement already satisfied: sniffio in /usr/local/lib/python3.10/dist-packages (from openai>=1.1.0->llama-index-core<0.11.0,>=0.10.1->llama-index-embeddings-huggingface) (1.3.1)\n",
            "Requirement already satisfied: certifi in /usr/local/lib/python3.10/dist-packages (from httpx->llama-index-core<0.11.0,>=0.10.1->llama-index-embeddings-huggingface) (2024.7.4)\n",
            "Requirement already satisfied: httpcore==1.* in /usr/local/lib/python3.10/dist-packages (from httpx->llama-index-core<0.11.0,>=0.10.1->llama-index-embeddings-huggingface) (1.0.5)\n",
            "Requirement already satisfied: idna in /usr/local/lib/python3.10/dist-packages (from httpx->llama-index-core<0.11.0,>=0.10.1->llama-index-embeddings-huggingface) (3.7)\n",
            "Requirement already satisfied: h11<0.15,>=0.13 in /usr/local/lib/python3.10/dist-packages (from httpcore==1.*->httpx->llama-index-core<0.11.0,>=0.10.1->llama-index-embeddings-huggingface) (0.14.0)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub[inference]>=0.19.0->llama-index-embeddings-huggingface) (3.3.2)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub[inference]>=0.19.0->llama-index-embeddings-huggingface) (1.26.19)\n",
            "Requirement already satisfied: greenlet!=0.4.17 in /usr/local/lib/python3.10/dist-packages (from SQLAlchemy[asyncio]>=1.4.49->llama-index-core<0.11.0,>=0.10.1->llama-index-embeddings-huggingface) (3.0.3)\n",
            "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch>=1.11.0->sentence-transformers>=2.6.1->llama-index-embeddings-huggingface) (1.13.0)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch>=1.11.0->sentence-transformers>=2.6.1->llama-index-embeddings-huggingface) (3.1.4)\n",
            "Requirement already satisfied: nvidia-cuda-nvrtc-cu12==12.1.105 in /usr/local/lib/python3.10/dist-packages (from torch>=1.11.0->sentence-transformers>=2.6.1->llama-index-embeddings-huggingface) (12.1.105)\n",
            "Requirement already satisfied: nvidia-cuda-runtime-cu12==12.1.105 in /usr/local/lib/python3.10/dist-packages (from torch>=1.11.0->sentence-transformers>=2.6.1->llama-index-embeddings-huggingface) (12.1.105)\n",
            "Requirement already satisfied: nvidia-cuda-cupti-cu12==12.1.105 in /usr/local/lib/python3.10/dist-packages (from torch>=1.11.0->sentence-transformers>=2.6.1->llama-index-embeddings-huggingface) (12.1.105)\n",
            "Requirement already satisfied: nvidia-cudnn-cu12==8.9.2.26 in /usr/local/lib/python3.10/dist-packages (from torch>=1.11.0->sentence-transformers>=2.6.1->llama-index-embeddings-huggingface) (8.9.2.26)\n",
            "Requirement already satisfied: nvidia-cublas-cu12==12.1.3.1 in /usr/local/lib/python3.10/dist-packages (from torch>=1.11.0->sentence-transformers>=2.6.1->llama-index-embeddings-huggingface) (12.1.3.1)\n",
            "Requirement already satisfied: nvidia-cufft-cu12==11.0.2.54 in /usr/local/lib/python3.10/dist-packages (from torch>=1.11.0->sentence-transformers>=2.6.1->llama-index-embeddings-huggingface) (11.0.2.54)\n",
            "Requirement already satisfied: nvidia-curand-cu12==10.3.2.106 in /usr/local/lib/python3.10/dist-packages (from torch>=1.11.0->sentence-transformers>=2.6.1->llama-index-embeddings-huggingface) (10.3.2.106)\n",
            "Requirement already satisfied: nvidia-cusolver-cu12==11.4.5.107 in /usr/local/lib/python3.10/dist-packages (from torch>=1.11.0->sentence-transformers>=2.6.1->llama-index-embeddings-huggingface) (11.4.5.107)\n",
            "Requirement already satisfied: nvidia-cusparse-cu12==12.1.0.106 in /usr/local/lib/python3.10/dist-packages (from torch>=1.11.0->sentence-transformers>=2.6.1->llama-index-embeddings-huggingface) (12.1.0.106)\n",
            "Requirement already satisfied: nvidia-nccl-cu12==2.20.5 in /usr/local/lib/python3.10/dist-packages (from torch>=1.11.0->sentence-transformers>=2.6.1->llama-index-embeddings-huggingface) (2.20.5)\n",
            "Requirement already satisfied: nvidia-nvtx-cu12==12.1.105 in /usr/local/lib/python3.10/dist-packages (from torch>=1.11.0->sentence-transformers>=2.6.1->llama-index-embeddings-huggingface) (12.1.105)\n",
            "Requirement already satisfied: triton==2.3.0 in /usr/local/lib/python3.10/dist-packages (from torch>=1.11.0->sentence-transformers>=2.6.1->llama-index-embeddings-huggingface) (2.3.0)\n",
            "Requirement already satisfied: nvidia-nvjitlink-cu12 in /usr/local/lib/python3.10/dist-packages (from nvidia-cusolver-cu12==11.4.5.107->torch>=1.11.0->sentence-transformers>=2.6.1->llama-index-embeddings-huggingface) (12.5.82)\n",
            "Requirement already satisfied: tokenizers<0.20,>=0.19 in /usr/local/lib/python3.10/dist-packages (from transformers<5.0.0,>=4.34.0->sentence-transformers>=2.6.1->llama-index-embeddings-huggingface) (0.19.1)\n",
            "Requirement already satisfied: safetensors>=0.4.1 in /usr/local/lib/python3.10/dist-packages (from transformers<5.0.0,>=4.34.0->sentence-transformers>=2.6.1->llama-index-embeddings-huggingface) (0.4.3)\n",
            "Requirement already satisfied: mypy-extensions>=0.3.0 in /usr/local/lib/python3.10/dist-packages (from typing-inspect>=0.8.0->llama-index-core<0.11.0,>=0.10.1->llama-index-embeddings-huggingface) (1.0.0)\n",
            "Requirement already satisfied: marshmallow<4.0.0,>=3.18.0 in /usr/local/lib/python3.10/dist-packages (from dataclasses-json->llama-index-core<0.11.0,>=0.10.1->llama-index-embeddings-huggingface) (3.21.3)\n",
            "Requirement already satisfied: python-dateutil>=2.8.2 in /usr/local/lib/python3.10/dist-packages (from pandas->llama-index-core<0.11.0,>=0.10.1->llama-index-embeddings-huggingface) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas->llama-index-core<0.11.0,>=0.10.1->llama-index-embeddings-huggingface) (2023.4)\n",
            "Requirement already satisfied: tzdata>=2022.1 in /usr/local/lib/python3.10/dist-packages (from pandas->llama-index-core<0.11.0,>=0.10.1->llama-index-embeddings-huggingface) (2024.1)\n",
            "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from scikit-learn->sentence-transformers>=2.6.1->llama-index-embeddings-huggingface) (3.5.0)\n",
            "Requirement already satisfied: exceptiongroup in /usr/local/lib/python3.10/dist-packages (from anyio<5,>=3.5.0->openai>=1.1.0->llama-index-core<0.11.0,>=0.10.1->llama-index-embeddings-huggingface) (1.2.1)\n",
            "Requirement already satisfied: annotated-types>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from pydantic<3,>=1.9.0->openai>=1.1.0->llama-index-core<0.11.0,>=0.10.1->llama-index-embeddings-huggingface) (0.7.0)\n",
            "Requirement already satisfied: pydantic-core==2.20.1 in /usr/local/lib/python3.10/dist-packages (from pydantic<3,>=1.9.0->openai>=1.1.0->llama-index-core<0.11.0,>=0.10.1->llama-index-embeddings-huggingface) (2.20.1)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.8.2->pandas->llama-index-core<0.11.0,>=0.10.1->llama-index-embeddings-huggingface) (1.16.0)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch>=1.11.0->sentence-transformers>=2.6.1->llama-index-embeddings-huggingface) (2.1.5)\n",
            "Requirement already satisfied: mpmath<1.4,>=1.1.0 in /usr/local/lib/python3.10/dist-packages (from sympy->torch>=1.11.0->sentence-transformers>=2.6.1->llama-index-embeddings-huggingface) (1.3.0)\n",
            "Requirement already satisfied: torch in /usr/local/lib/python3.10/dist-packages (2.3.0+cu121)\n",
            "Requirement already satisfied: sentence-transformers in /usr/local/lib/python3.10/dist-packages (3.0.1)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from torch) (3.15.4)\n",
            "Requirement already satisfied: typing-extensions>=4.8.0 in /usr/local/lib/python3.10/dist-packages (from torch) (4.12.2)\n",
            "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch) (1.13.0)\n",
            "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch) (3.3)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch) (3.1.4)\n",
            "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from torch) (2023.6.0)\n",
            "Requirement already satisfied: nvidia-cuda-nvrtc-cu12==12.1.105 in /usr/local/lib/python3.10/dist-packages (from torch) (12.1.105)\n",
            "Requirement already satisfied: nvidia-cuda-runtime-cu12==12.1.105 in /usr/local/lib/python3.10/dist-packages (from torch) (12.1.105)\n",
            "Requirement already satisfied: nvidia-cuda-cupti-cu12==12.1.105 in /usr/local/lib/python3.10/dist-packages (from torch) (12.1.105)\n",
            "Requirement already satisfied: nvidia-cudnn-cu12==8.9.2.26 in /usr/local/lib/python3.10/dist-packages (from torch) (8.9.2.26)\n",
            "Requirement already satisfied: nvidia-cublas-cu12==12.1.3.1 in /usr/local/lib/python3.10/dist-packages (from torch) (12.1.3.1)\n",
            "Requirement already satisfied: nvidia-cufft-cu12==11.0.2.54 in /usr/local/lib/python3.10/dist-packages (from torch) (11.0.2.54)\n",
            "Requirement already satisfied: nvidia-curand-cu12==10.3.2.106 in /usr/local/lib/python3.10/dist-packages (from torch) (10.3.2.106)\n",
            "Requirement already satisfied: nvidia-cusolver-cu12==11.4.5.107 in /usr/local/lib/python3.10/dist-packages (from torch) (11.4.5.107)\n",
            "Requirement already satisfied: nvidia-cusparse-cu12==12.1.0.106 in /usr/local/lib/python3.10/dist-packages (from torch) (12.1.0.106)\n",
            "Requirement already satisfied: nvidia-nccl-cu12==2.20.5 in /usr/local/lib/python3.10/dist-packages (from torch) (2.20.5)\n",
            "Requirement already satisfied: nvidia-nvtx-cu12==12.1.105 in /usr/local/lib/python3.10/dist-packages (from torch) (12.1.105)\n",
            "Requirement already satisfied: triton==2.3.0 in /usr/local/lib/python3.10/dist-packages (from torch) (2.3.0)\n",
            "Requirement already satisfied: nvidia-nvjitlink-cu12 in /usr/local/lib/python3.10/dist-packages (from nvidia-cusolver-cu12==11.4.5.107->torch) (12.5.82)\n",
            "Requirement already satisfied: transformers<5.0.0,>=4.34.0 in /usr/local/lib/python3.10/dist-packages (from sentence-transformers) (4.41.2)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.10/dist-packages (from sentence-transformers) (4.66.4)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from sentence-transformers) (1.25.2)\n",
            "Requirement already satisfied: scikit-learn in /usr/local/lib/python3.10/dist-packages (from sentence-transformers) (1.2.2)\n",
            "Requirement already satisfied: scipy in /usr/local/lib/python3.10/dist-packages (from sentence-transformers) (1.11.4)\n",
            "Requirement already satisfied: huggingface-hub>=0.15.1 in /usr/local/lib/python3.10/dist-packages (from sentence-transformers) (0.23.4)\n",
            "Requirement already satisfied: Pillow in /usr/local/lib/python3.10/dist-packages (from sentence-transformers) (9.4.0)\n",
            "Requirement already satisfied: packaging>=20.9 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.15.1->sentence-transformers) (24.1)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.15.1->sentence-transformers) (6.0.1)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.15.1->sentence-transformers) (2.31.0)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers<5.0.0,>=4.34.0->sentence-transformers) (2024.5.15)\n",
            "Requirement already satisfied: tokenizers<0.20,>=0.19 in /usr/local/lib/python3.10/dist-packages (from transformers<5.0.0,>=4.34.0->sentence-transformers) (0.19.1)\n",
            "Requirement already satisfied: safetensors>=0.4.1 in /usr/local/lib/python3.10/dist-packages (from transformers<5.0.0,>=4.34.0->sentence-transformers) (0.4.3)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch) (2.1.5)\n",
            "Requirement already satisfied: joblib>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from scikit-learn->sentence-transformers) (1.4.2)\n",
            "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from scikit-learn->sentence-transformers) (3.5.0)\n",
            "Requirement already satisfied: mpmath<1.4,>=1.1.0 in /usr/local/lib/python3.10/dist-packages (from sympy->torch) (1.3.0)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub>=0.15.1->sentence-transformers) (3.3.2)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub>=0.15.1->sentence-transformers) (3.7)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub>=0.15.1->sentence-transformers) (1.26.19)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub>=0.15.1->sentence-transformers) (2024.7.4)\n",
            "Requirement already satisfied: llama_index in /usr/local/lib/python3.10/dist-packages (0.10.55)\n",
            "Requirement already satisfied: llama-index-agent-openai<0.3.0,>=0.1.4 in /usr/local/lib/python3.10/dist-packages (from llama_index) (0.2.8)\n",
            "Requirement already satisfied: llama-index-cli<0.2.0,>=0.1.2 in /usr/local/lib/python3.10/dist-packages (from llama_index) (0.1.12)\n",
            "Requirement already satisfied: llama-index-core==0.10.55 in /usr/local/lib/python3.10/dist-packages (from llama_index) (0.10.55)\n",
            "Requirement already satisfied: llama-index-embeddings-openai<0.2.0,>=0.1.5 in /usr/local/lib/python3.10/dist-packages (from llama_index) (0.1.10)\n",
            "Requirement already satisfied: llama-index-indices-managed-llama-cloud>=0.2.0 in /usr/local/lib/python3.10/dist-packages (from llama_index) (0.2.5)\n",
            "Requirement already satisfied: llama-index-legacy<0.10.0,>=0.9.48 in /usr/local/lib/python3.10/dist-packages (from llama_index) (0.9.48)\n",
            "Requirement already satisfied: llama-index-llms-openai<0.2.0,>=0.1.13 in /usr/local/lib/python3.10/dist-packages (from llama_index) (0.1.25)\n",
            "Requirement already satisfied: llama-index-multi-modal-llms-openai<0.2.0,>=0.1.3 in /usr/local/lib/python3.10/dist-packages (from llama_index) (0.1.7)\n",
            "Requirement already satisfied: llama-index-program-openai<0.2.0,>=0.1.3 in /usr/local/lib/python3.10/dist-packages (from llama_index) (0.1.6)\n",
            "Requirement already satisfied: llama-index-question-gen-openai<0.2.0,>=0.1.2 in /usr/local/lib/python3.10/dist-packages (from llama_index) (0.1.3)\n",
            "Requirement already satisfied: llama-index-readers-file<0.2.0,>=0.1.4 in /usr/local/lib/python3.10/dist-packages (from llama_index) (0.1.30)\n",
            "Requirement already satisfied: llama-index-readers-llama-parse>=0.1.2 in /usr/local/lib/python3.10/dist-packages (from llama_index) (0.1.6)\n",
            "Requirement already satisfied: PyYAML>=6.0.1 in /usr/local/lib/python3.10/dist-packages (from llama-index-core==0.10.55->llama_index) (6.0.1)\n",
            "Requirement already satisfied: SQLAlchemy[asyncio]>=1.4.49 in /usr/local/lib/python3.10/dist-packages (from llama-index-core==0.10.55->llama_index) (2.0.31)\n",
            "Requirement already satisfied: aiohttp<4.0.0,>=3.8.6 in /usr/local/lib/python3.10/dist-packages (from llama-index-core==0.10.55->llama_index) (3.9.5)\n",
            "Requirement already satisfied: dataclasses-json in /usr/local/lib/python3.10/dist-packages (from llama-index-core==0.10.55->llama_index) (0.6.7)\n",
            "Requirement already satisfied: deprecated>=1.2.9.3 in /usr/local/lib/python3.10/dist-packages (from llama-index-core==0.10.55->llama_index) (1.2.14)\n",
            "Requirement already satisfied: dirtyjson<2.0.0,>=1.0.8 in /usr/local/lib/python3.10/dist-packages (from llama-index-core==0.10.55->llama_index) (1.0.8)\n",
            "Requirement already satisfied: fsspec>=2023.5.0 in /usr/local/lib/python3.10/dist-packages (from llama-index-core==0.10.55->llama_index) (2023.6.0)\n",
            "Requirement already satisfied: httpx in /usr/local/lib/python3.10/dist-packages (from llama-index-core==0.10.55->llama_index) (0.27.0)\n",
            "Requirement already satisfied: nest-asyncio<2.0.0,>=1.5.8 in /usr/local/lib/python3.10/dist-packages (from llama-index-core==0.10.55->llama_index) (1.6.0)\n",
            "Requirement already satisfied: networkx>=3.0 in /usr/local/lib/python3.10/dist-packages (from llama-index-core==0.10.55->llama_index) (3.3)\n",
            "Requirement already satisfied: nltk<4.0.0,>=3.8.1 in /usr/local/lib/python3.10/dist-packages (from llama-index-core==0.10.55->llama_index) (3.8.1)\n",
            "Requirement already satisfied: numpy<2.0.0 in /usr/local/lib/python3.10/dist-packages (from llama-index-core==0.10.55->llama_index) (1.25.2)\n",
            "Requirement already satisfied: openai>=1.1.0 in /usr/local/lib/python3.10/dist-packages (from llama-index-core==0.10.55->llama_index) (1.35.14)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from llama-index-core==0.10.55->llama_index) (2.0.3)\n",
            "Requirement already satisfied: pillow>=9.0.0 in /usr/local/lib/python3.10/dist-packages (from llama-index-core==0.10.55->llama_index) (9.4.0)\n",
            "Requirement already satisfied: requests>=2.31.0 in /usr/local/lib/python3.10/dist-packages (from llama-index-core==0.10.55->llama_index) (2.31.0)\n",
            "Requirement already satisfied: tenacity!=8.4.0,<9.0.0,>=8.2.0 in /usr/local/lib/python3.10/dist-packages (from llama-index-core==0.10.55->llama_index) (8.5.0)\n",
            "Requirement already satisfied: tiktoken>=0.3.3 in /usr/local/lib/python3.10/dist-packages (from llama-index-core==0.10.55->llama_index) (0.7.0)\n",
            "Requirement already satisfied: tqdm<5.0.0,>=4.66.1 in /usr/local/lib/python3.10/dist-packages (from llama-index-core==0.10.55->llama_index) (4.66.4)\n",
            "Requirement already satisfied: typing-extensions>=4.5.0 in /usr/local/lib/python3.10/dist-packages (from llama-index-core==0.10.55->llama_index) (4.12.2)\n",
            "Requirement already satisfied: typing-inspect>=0.8.0 in /usr/local/lib/python3.10/dist-packages (from llama-index-core==0.10.55->llama_index) (0.9.0)\n",
            "Requirement already satisfied: wrapt in /usr/local/lib/python3.10/dist-packages (from llama-index-core==0.10.55->llama_index) (1.14.1)\n",
            "Requirement already satisfied: llama-cloud>=0.0.9 in /usr/local/lib/python3.10/dist-packages (from llama-index-indices-managed-llama-cloud>=0.2.0->llama_index) (0.0.9)\n",
            "Requirement already satisfied: beautifulsoup4<5.0.0,>=4.12.3 in /usr/local/lib/python3.10/dist-packages (from llama-index-readers-file<0.2.0,>=0.1.4->llama_index) (4.12.3)\n",
            "Requirement already satisfied: pypdf<5.0.0,>=4.0.1 in /usr/local/lib/python3.10/dist-packages (from llama-index-readers-file<0.2.0,>=0.1.4->llama_index) (4.3.0)\n",
            "Requirement already satisfied: striprtf<0.0.27,>=0.0.26 in /usr/local/lib/python3.10/dist-packages (from llama-index-readers-file<0.2.0,>=0.1.4->llama_index) (0.0.26)\n",
            "Requirement already satisfied: llama-parse>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from llama-index-readers-llama-parse>=0.1.2->llama_index) (0.4.8)\n",
            "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.6->llama-index-core==0.10.55->llama_index) (1.3.1)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.6->llama-index-core==0.10.55->llama_index) (23.2.0)\n",
            "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.6->llama-index-core==0.10.55->llama_index) (1.4.1)\n",
            "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.6->llama-index-core==0.10.55->llama_index) (6.0.5)\n",
            "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.6->llama-index-core==0.10.55->llama_index) (1.9.4)\n",
            "Requirement already satisfied: async-timeout<5.0,>=4.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.6->llama-index-core==0.10.55->llama_index) (4.0.3)\n",
            "Requirement already satisfied: soupsieve>1.2 in /usr/local/lib/python3.10/dist-packages (from beautifulsoup4<5.0.0,>=4.12.3->llama-index-readers-file<0.2.0,>=0.1.4->llama_index) (2.5)\n",
            "Requirement already satisfied: pydantic>=1.10 in /usr/local/lib/python3.10/dist-packages (from llama-cloud>=0.0.9->llama-index-indices-managed-llama-cloud>=0.2.0->llama_index) (2.8.2)\n",
            "Requirement already satisfied: anyio in /usr/local/lib/python3.10/dist-packages (from httpx->llama-index-core==0.10.55->llama_index) (3.7.1)\n",
            "Requirement already satisfied: certifi in /usr/local/lib/python3.10/dist-packages (from httpx->llama-index-core==0.10.55->llama_index) (2024.7.4)\n",
            "Requirement already satisfied: httpcore==1.* in /usr/local/lib/python3.10/dist-packages (from httpx->llama-index-core==0.10.55->llama_index) (1.0.5)\n",
            "Requirement already satisfied: idna in /usr/local/lib/python3.10/dist-packages (from httpx->llama-index-core==0.10.55->llama_index) (3.7)\n",
            "Requirement already satisfied: sniffio in /usr/local/lib/python3.10/dist-packages (from httpx->llama-index-core==0.10.55->llama_index) (1.3.1)\n",
            "Requirement already satisfied: h11<0.15,>=0.13 in /usr/local/lib/python3.10/dist-packages (from httpcore==1.*->httpx->llama-index-core==0.10.55->llama_index) (0.14.0)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.10/dist-packages (from nltk<4.0.0,>=3.8.1->llama-index-core==0.10.55->llama_index) (8.1.7)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.10/dist-packages (from nltk<4.0.0,>=3.8.1->llama-index-core==0.10.55->llama_index) (1.4.2)\n",
            "Requirement already satisfied: regex>=2021.8.3 in /usr/local/lib/python3.10/dist-packages (from nltk<4.0.0,>=3.8.1->llama-index-core==0.10.55->llama_index) (2024.5.15)\n",
            "Requirement already satisfied: distro<2,>=1.7.0 in /usr/lib/python3/dist-packages (from openai>=1.1.0->llama-index-core==0.10.55->llama_index) (1.7.0)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests>=2.31.0->llama-index-core==0.10.55->llama_index) (3.3.2)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests>=2.31.0->llama-index-core==0.10.55->llama_index) (1.26.19)\n",
            "Requirement already satisfied: greenlet!=0.4.17 in /usr/local/lib/python3.10/dist-packages (from SQLAlchemy[asyncio]>=1.4.49->llama-index-core==0.10.55->llama_index) (3.0.3)\n",
            "Requirement already satisfied: mypy-extensions>=0.3.0 in /usr/local/lib/python3.10/dist-packages (from typing-inspect>=0.8.0->llama-index-core==0.10.55->llama_index) (1.0.0)\n",
            "Requirement already satisfied: marshmallow<4.0.0,>=3.18.0 in /usr/local/lib/python3.10/dist-packages (from dataclasses-json->llama-index-core==0.10.55->llama_index) (3.21.3)\n",
            "Requirement already satisfied: python-dateutil>=2.8.2 in /usr/local/lib/python3.10/dist-packages (from pandas->llama-index-core==0.10.55->llama_index) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas->llama-index-core==0.10.55->llama_index) (2023.4)\n",
            "Requirement already satisfied: tzdata>=2022.1 in /usr/local/lib/python3.10/dist-packages (from pandas->llama-index-core==0.10.55->llama_index) (2024.1)\n",
            "Requirement already satisfied: exceptiongroup in /usr/local/lib/python3.10/dist-packages (from anyio->httpx->llama-index-core==0.10.55->llama_index) (1.2.1)\n",
            "Requirement already satisfied: packaging>=17.0 in /usr/local/lib/python3.10/dist-packages (from marshmallow<4.0.0,>=3.18.0->dataclasses-json->llama-index-core==0.10.55->llama_index) (24.1)\n",
            "Requirement already satisfied: annotated-types>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from pydantic>=1.10->llama-cloud>=0.0.9->llama-index-indices-managed-llama-cloud>=0.2.0->llama_index) (0.7.0)\n",
            "Requirement already satisfied: pydantic-core==2.20.1 in /usr/local/lib/python3.10/dist-packages (from pydantic>=1.10->llama-cloud>=0.0.9->llama-index-indices-managed-llama-cloud>=0.2.0->llama_index) (2.20.1)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.8.2->pandas->llama-index-core==0.10.55->llama_index) (1.16.0)\n",
            "Requirement already satisfied: tensorflow-io in /usr/local/lib/python3.10/dist-packages (0.37.1)\n",
            "Requirement already satisfied: tensorflow-io-gcs-filesystem==0.37.1 in /usr/local/lib/python3.10/dist-packages (from tensorflow-io) (0.37.1)\n",
            "Requirement already satisfied: llama-index-vector-stores-chroma in /usr/local/lib/python3.10/dist-packages (0.1.10)\n",
            "Requirement already satisfied: chromadb<0.6.0,>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from llama-index-vector-stores-chroma) (0.5.4)\n",
            "Requirement already satisfied: llama-index-core<0.11.0,>=0.10.1 in /usr/local/lib/python3.10/dist-packages (from llama-index-vector-stores-chroma) (0.10.55)\n",
            "Requirement already satisfied: build>=1.0.3 in /usr/local/lib/python3.10/dist-packages (from chromadb<0.6.0,>=0.4.0->llama-index-vector-stores-chroma) (1.2.1)\n",
            "Requirement already satisfied: pydantic>=1.9 in /usr/local/lib/python3.10/dist-packages (from chromadb<0.6.0,>=0.4.0->llama-index-vector-stores-chroma) (2.8.2)\n",
            "Requirement already satisfied: chroma-hnswlib==0.7.5 in /usr/local/lib/python3.10/dist-packages (from chromadb<0.6.0,>=0.4.0->llama-index-vector-stores-chroma) (0.7.5)\n",
            "Requirement already satisfied: fastapi>=0.95.2 in /usr/local/lib/python3.10/dist-packages (from chromadb<0.6.0,>=0.4.0->llama-index-vector-stores-chroma) (0.111.1)\n",
            "Requirement already satisfied: uvicorn[standard]>=0.18.3 in /usr/local/lib/python3.10/dist-packages (from chromadb<0.6.0,>=0.4.0->llama-index-vector-stores-chroma) (0.30.1)\n",
            "Requirement already satisfied: numpy<2.0.0,>=1.22.5 in /usr/local/lib/python3.10/dist-packages (from chromadb<0.6.0,>=0.4.0->llama-index-vector-stores-chroma) (1.25.2)\n",
            "Requirement already satisfied: posthog>=2.4.0 in /usr/local/lib/python3.10/dist-packages (from chromadb<0.6.0,>=0.4.0->llama-index-vector-stores-chroma) (3.5.0)\n",
            "Requirement already satisfied: typing-extensions>=4.5.0 in /usr/local/lib/python3.10/dist-packages (from chromadb<0.6.0,>=0.4.0->llama-index-vector-stores-chroma) (4.12.2)\n",
            "Requirement already satisfied: onnxruntime>=1.14.1 in /usr/local/lib/python3.10/dist-packages (from chromadb<0.6.0,>=0.4.0->llama-index-vector-stores-chroma) (1.18.1)\n",
            "Requirement already satisfied: opentelemetry-api>=1.2.0 in /usr/local/lib/python3.10/dist-packages (from chromadb<0.6.0,>=0.4.0->llama-index-vector-stores-chroma) (1.25.0)\n",
            "Requirement already satisfied: opentelemetry-exporter-otlp-proto-grpc>=1.2.0 in /usr/local/lib/python3.10/dist-packages (from chromadb<0.6.0,>=0.4.0->llama-index-vector-stores-chroma) (1.25.0)\n",
            "Requirement already satisfied: opentelemetry-instrumentation-fastapi>=0.41b0 in /usr/local/lib/python3.10/dist-packages (from chromadb<0.6.0,>=0.4.0->llama-index-vector-stores-chroma) (0.46b0)\n",
            "Requirement already satisfied: opentelemetry-sdk>=1.2.0 in /usr/local/lib/python3.10/dist-packages (from chromadb<0.6.0,>=0.4.0->llama-index-vector-stores-chroma) (1.25.0)\n",
            "Requirement already satisfied: tokenizers>=0.13.2 in /usr/local/lib/python3.10/dist-packages (from chromadb<0.6.0,>=0.4.0->llama-index-vector-stores-chroma) (0.19.1)\n",
            "Requirement already satisfied: pypika>=0.48.9 in /usr/local/lib/python3.10/dist-packages (from chromadb<0.6.0,>=0.4.0->llama-index-vector-stores-chroma) (0.48.9)\n",
            "Requirement already satisfied: tqdm>=4.65.0 in /usr/local/lib/python3.10/dist-packages (from chromadb<0.6.0,>=0.4.0->llama-index-vector-stores-chroma) (4.66.4)\n",
            "Requirement already satisfied: overrides>=7.3.1 in /usr/local/lib/python3.10/dist-packages (from chromadb<0.6.0,>=0.4.0->llama-index-vector-stores-chroma) (7.7.0)\n",
            "Requirement already satisfied: importlib-resources in /usr/local/lib/python3.10/dist-packages (from chromadb<0.6.0,>=0.4.0->llama-index-vector-stores-chroma) (6.4.0)\n",
            "Requirement already satisfied: grpcio>=1.58.0 in /usr/local/lib/python3.10/dist-packages (from chromadb<0.6.0,>=0.4.0->llama-index-vector-stores-chroma) (1.64.1)\n",
            "Requirement already satisfied: bcrypt>=4.0.1 in /usr/local/lib/python3.10/dist-packages (from chromadb<0.6.0,>=0.4.0->llama-index-vector-stores-chroma) (4.1.3)\n",
            "Requirement already satisfied: typer>=0.9.0 in /usr/local/lib/python3.10/dist-packages (from chromadb<0.6.0,>=0.4.0->llama-index-vector-stores-chroma) (0.12.3)\n",
            "Requirement already satisfied: kubernetes>=28.1.0 in /usr/local/lib/python3.10/dist-packages (from chromadb<0.6.0,>=0.4.0->llama-index-vector-stores-chroma) (30.1.0)\n",
            "Requirement already satisfied: tenacity>=8.2.3 in /usr/local/lib/python3.10/dist-packages (from chromadb<0.6.0,>=0.4.0->llama-index-vector-stores-chroma) (8.5.0)\n",
            "Requirement already satisfied: PyYAML>=6.0.0 in /usr/local/lib/python3.10/dist-packages (from chromadb<0.6.0,>=0.4.0->llama-index-vector-stores-chroma) (6.0.1)\n",
            "Requirement already satisfied: mmh3>=4.0.1 in /usr/local/lib/python3.10/dist-packages (from chromadb<0.6.0,>=0.4.0->llama-index-vector-stores-chroma) (4.1.0)\n",
            "Requirement already satisfied: orjson>=3.9.12 in /usr/local/lib/python3.10/dist-packages (from chromadb<0.6.0,>=0.4.0->llama-index-vector-stores-chroma) (3.10.6)\n",
            "Requirement already satisfied: httpx>=0.27.0 in /usr/local/lib/python3.10/dist-packages (from chromadb<0.6.0,>=0.4.0->llama-index-vector-stores-chroma) (0.27.0)\n",
            "Requirement already satisfied: SQLAlchemy[asyncio]>=1.4.49 in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-vector-stores-chroma) (2.0.31)\n",
            "Requirement already satisfied: aiohttp<4.0.0,>=3.8.6 in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-vector-stores-chroma) (3.9.5)\n",
            "Requirement already satisfied: dataclasses-json in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-vector-stores-chroma) (0.6.7)\n",
            "Requirement already satisfied: deprecated>=1.2.9.3 in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-vector-stores-chroma) (1.2.14)\n",
            "Requirement already satisfied: dirtyjson<2.0.0,>=1.0.8 in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-vector-stores-chroma) (1.0.8)\n",
            "Requirement already satisfied: fsspec>=2023.5.0 in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-vector-stores-chroma) (2023.6.0)\n",
            "Requirement already satisfied: nest-asyncio<2.0.0,>=1.5.8 in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-vector-stores-chroma) (1.6.0)\n",
            "Requirement already satisfied: networkx>=3.0 in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-vector-stores-chroma) (3.3)\n",
            "Requirement already satisfied: nltk<4.0.0,>=3.8.1 in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-vector-stores-chroma) (3.8.1)\n",
            "Requirement already satisfied: openai>=1.1.0 in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-vector-stores-chroma) (1.35.14)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-vector-stores-chroma) (2.0.3)\n",
            "Requirement already satisfied: pillow>=9.0.0 in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-vector-stores-chroma) (9.4.0)\n",
            "Requirement already satisfied: requests>=2.31.0 in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-vector-stores-chroma) (2.31.0)\n",
            "Requirement already satisfied: tiktoken>=0.3.3 in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-vector-stores-chroma) (0.7.0)\n",
            "Requirement already satisfied: typing-inspect>=0.8.0 in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-vector-stores-chroma) (0.9.0)\n",
            "Requirement already satisfied: wrapt in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.11.0,>=0.10.1->llama-index-vector-stores-chroma) (1.14.1)\n",
            "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.6->llama-index-core<0.11.0,>=0.10.1->llama-index-vector-stores-chroma) (1.3.1)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.6->llama-index-core<0.11.0,>=0.10.1->llama-index-vector-stores-chroma) (23.2.0)\n",
            "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.6->llama-index-core<0.11.0,>=0.10.1->llama-index-vector-stores-chroma) (1.4.1)\n",
            "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.6->llama-index-core<0.11.0,>=0.10.1->llama-index-vector-stores-chroma) (6.0.5)\n",
            "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.6->llama-index-core<0.11.0,>=0.10.1->llama-index-vector-stores-chroma) (1.9.4)\n",
            "Requirement already satisfied: async-timeout<5.0,>=4.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.6->llama-index-core<0.11.0,>=0.10.1->llama-index-vector-stores-chroma) (4.0.3)\n",
            "Requirement already satisfied: packaging>=19.1 in /usr/local/lib/python3.10/dist-packages (from build>=1.0.3->chromadb<0.6.0,>=0.4.0->llama-index-vector-stores-chroma) (24.1)\n",
            "Requirement already satisfied: pyproject_hooks in /usr/local/lib/python3.10/dist-packages (from build>=1.0.3->chromadb<0.6.0,>=0.4.0->llama-index-vector-stores-chroma) (1.1.0)\n",
            "Requirement already satisfied: tomli>=1.1.0 in /usr/local/lib/python3.10/dist-packages (from build>=1.0.3->chromadb<0.6.0,>=0.4.0->llama-index-vector-stores-chroma) (2.0.1)\n",
            "Requirement already satisfied: starlette<0.38.0,>=0.37.2 in /usr/local/lib/python3.10/dist-packages (from fastapi>=0.95.2->chromadb<0.6.0,>=0.4.0->llama-index-vector-stores-chroma) (0.37.2)\n",
            "Requirement already satisfied: fastapi-cli>=0.0.2 in /usr/local/lib/python3.10/dist-packages (from fastapi>=0.95.2->chromadb<0.6.0,>=0.4.0->llama-index-vector-stores-chroma) (0.0.4)\n",
            "Requirement already satisfied: jinja2>=2.11.2 in /usr/local/lib/python3.10/dist-packages (from fastapi>=0.95.2->chromadb<0.6.0,>=0.4.0->llama-index-vector-stores-chroma) (3.1.4)\n",
            "Requirement already satisfied: python-multipart>=0.0.7 in /usr/local/lib/python3.10/dist-packages (from fastapi>=0.95.2->chromadb<0.6.0,>=0.4.0->llama-index-vector-stores-chroma) (0.0.9)\n",
            "Requirement already satisfied: email_validator>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from fastapi>=0.95.2->chromadb<0.6.0,>=0.4.0->llama-index-vector-stores-chroma) (2.2.0)\n",
            "Requirement already satisfied: anyio in /usr/local/lib/python3.10/dist-packages (from httpx>=0.27.0->chromadb<0.6.0,>=0.4.0->llama-index-vector-stores-chroma) (3.7.1)\n",
            "Requirement already satisfied: certifi in /usr/local/lib/python3.10/dist-packages (from httpx>=0.27.0->chromadb<0.6.0,>=0.4.0->llama-index-vector-stores-chroma) (2024.7.4)\n",
            "Requirement already satisfied: httpcore==1.* in /usr/local/lib/python3.10/dist-packages (from httpx>=0.27.0->chromadb<0.6.0,>=0.4.0->llama-index-vector-stores-chroma) (1.0.5)\n",
            "Requirement already satisfied: idna in /usr/local/lib/python3.10/dist-packages (from httpx>=0.27.0->chromadb<0.6.0,>=0.4.0->llama-index-vector-stores-chroma) (3.7)\n",
            "Requirement already satisfied: sniffio in /usr/local/lib/python3.10/dist-packages (from httpx>=0.27.0->chromadb<0.6.0,>=0.4.0->llama-index-vector-stores-chroma) (1.3.1)\n",
            "Requirement already satisfied: h11<0.15,>=0.13 in /usr/local/lib/python3.10/dist-packages (from httpcore==1.*->httpx>=0.27.0->chromadb<0.6.0,>=0.4.0->llama-index-vector-stores-chroma) (0.14.0)\n",
            "Requirement already satisfied: six>=1.9.0 in /usr/local/lib/python3.10/dist-packages (from kubernetes>=28.1.0->chromadb<0.6.0,>=0.4.0->llama-index-vector-stores-chroma) (1.16.0)\n",
            "Requirement already satisfied: python-dateutil>=2.5.3 in /usr/local/lib/python3.10/dist-packages (from kubernetes>=28.1.0->chromadb<0.6.0,>=0.4.0->llama-index-vector-stores-chroma) (2.8.2)\n",
            "Requirement already satisfied: google-auth>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from kubernetes>=28.1.0->chromadb<0.6.0,>=0.4.0->llama-index-vector-stores-chroma) (2.27.0)\n",
            "Requirement already satisfied: websocket-client!=0.40.0,!=0.41.*,!=0.42.*,>=0.32.0 in /usr/local/lib/python3.10/dist-packages (from kubernetes>=28.1.0->chromadb<0.6.0,>=0.4.0->llama-index-vector-stores-chroma) (1.8.0)\n",
            "Requirement already satisfied: requests-oauthlib in /usr/local/lib/python3.10/dist-packages (from kubernetes>=28.1.0->chromadb<0.6.0,>=0.4.0->llama-index-vector-stores-chroma) (1.3.1)\n",
            "Requirement already satisfied: oauthlib>=3.2.2 in /usr/local/lib/python3.10/dist-packages (from kubernetes>=28.1.0->chromadb<0.6.0,>=0.4.0->llama-index-vector-stores-chroma) (3.2.2)\n",
            "Requirement already satisfied: urllib3>=1.24.2 in /usr/local/lib/python3.10/dist-packages (from kubernetes>=28.1.0->chromadb<0.6.0,>=0.4.0->llama-index-vector-stores-chroma) (1.26.19)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.10/dist-packages (from nltk<4.0.0,>=3.8.1->llama-index-core<0.11.0,>=0.10.1->llama-index-vector-stores-chroma) (8.1.7)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.10/dist-packages (from nltk<4.0.0,>=3.8.1->llama-index-core<0.11.0,>=0.10.1->llama-index-vector-stores-chroma) (1.4.2)\n",
            "Requirement already satisfied: regex>=2021.8.3 in /usr/local/lib/python3.10/dist-packages (from nltk<4.0.0,>=3.8.1->llama-index-core<0.11.0,>=0.10.1->llama-index-vector-stores-chroma) (2024.5.15)\n",
            "Requirement already satisfied: coloredlogs in /usr/local/lib/python3.10/dist-packages (from onnxruntime>=1.14.1->chromadb<0.6.0,>=0.4.0->llama-index-vector-stores-chroma) (15.0.1)\n",
            "Requirement already satisfied: flatbuffers in /usr/local/lib/python3.10/dist-packages (from onnxruntime>=1.14.1->chromadb<0.6.0,>=0.4.0->llama-index-vector-stores-chroma) (24.3.25)\n",
            "Requirement already satisfied: protobuf in /usr/local/lib/python3.10/dist-packages (from onnxruntime>=1.14.1->chromadb<0.6.0,>=0.4.0->llama-index-vector-stores-chroma) (3.20.3)\n",
            "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from onnxruntime>=1.14.1->chromadb<0.6.0,>=0.4.0->llama-index-vector-stores-chroma) (1.13.0)\n",
            "Requirement already satisfied: distro<2,>=1.7.0 in /usr/lib/python3/dist-packages (from openai>=1.1.0->llama-index-core<0.11.0,>=0.10.1->llama-index-vector-stores-chroma) (1.7.0)\n",
            "Requirement already satisfied: importlib-metadata<=7.1,>=6.0 in /usr/local/lib/python3.10/dist-packages (from opentelemetry-api>=1.2.0->chromadb<0.6.0,>=0.4.0->llama-index-vector-stores-chroma) (7.1.0)\n",
            "Requirement already satisfied: googleapis-common-protos~=1.52 in /usr/local/lib/python3.10/dist-packages (from opentelemetry-exporter-otlp-proto-grpc>=1.2.0->chromadb<0.6.0,>=0.4.0->llama-index-vector-stores-chroma) (1.63.2)\n",
            "Requirement already satisfied: opentelemetry-exporter-otlp-proto-common==1.25.0 in /usr/local/lib/python3.10/dist-packages (from opentelemetry-exporter-otlp-proto-grpc>=1.2.0->chromadb<0.6.0,>=0.4.0->llama-index-vector-stores-chroma) (1.25.0)\n",
            "Requirement already satisfied: opentelemetry-proto==1.25.0 in /usr/local/lib/python3.10/dist-packages (from opentelemetry-exporter-otlp-proto-grpc>=1.2.0->chromadb<0.6.0,>=0.4.0->llama-index-vector-stores-chroma) (1.25.0)\n",
            "Requirement already satisfied: opentelemetry-instrumentation-asgi==0.46b0 in /usr/local/lib/python3.10/dist-packages (from opentelemetry-instrumentation-fastapi>=0.41b0->chromadb<0.6.0,>=0.4.0->llama-index-vector-stores-chroma) (0.46b0)\n",
            "Requirement already satisfied: opentelemetry-instrumentation==0.46b0 in /usr/local/lib/python3.10/dist-packages (from opentelemetry-instrumentation-fastapi>=0.41b0->chromadb<0.6.0,>=0.4.0->llama-index-vector-stores-chroma) (0.46b0)\n",
            "Requirement already satisfied: opentelemetry-semantic-conventions==0.46b0 in /usr/local/lib/python3.10/dist-packages (from opentelemetry-instrumentation-fastapi>=0.41b0->chromadb<0.6.0,>=0.4.0->llama-index-vector-stores-chroma) (0.46b0)\n",
            "Requirement already satisfied: opentelemetry-util-http==0.46b0 in /usr/local/lib/python3.10/dist-packages (from opentelemetry-instrumentation-fastapi>=0.41b0->chromadb<0.6.0,>=0.4.0->llama-index-vector-stores-chroma) (0.46b0)\n",
            "Requirement already satisfied: setuptools>=16.0 in /usr/local/lib/python3.10/dist-packages (from opentelemetry-instrumentation==0.46b0->opentelemetry-instrumentation-fastapi>=0.41b0->chromadb<0.6.0,>=0.4.0->llama-index-vector-stores-chroma) (67.7.2)\n",
            "Requirement already satisfied: asgiref~=3.0 in /usr/local/lib/python3.10/dist-packages (from opentelemetry-instrumentation-asgi==0.46b0->opentelemetry-instrumentation-fastapi>=0.41b0->chromadb<0.6.0,>=0.4.0->llama-index-vector-stores-chroma) (3.8.1)\n",
            "Requirement already satisfied: monotonic>=1.5 in /usr/local/lib/python3.10/dist-packages (from posthog>=2.4.0->chromadb<0.6.0,>=0.4.0->llama-index-vector-stores-chroma) (1.6)\n",
            "Requirement already satisfied: backoff>=1.10.0 in /usr/local/lib/python3.10/dist-packages (from posthog>=2.4.0->chromadb<0.6.0,>=0.4.0->llama-index-vector-stores-chroma) (2.2.1)\n",
            "Requirement already satisfied: annotated-types>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from pydantic>=1.9->chromadb<0.6.0,>=0.4.0->llama-index-vector-stores-chroma) (0.7.0)\n",
            "Requirement already satisfied: pydantic-core==2.20.1 in /usr/local/lib/python3.10/dist-packages (from pydantic>=1.9->chromadb<0.6.0,>=0.4.0->llama-index-vector-stores-chroma) (2.20.1)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests>=2.31.0->llama-index-core<0.11.0,>=0.10.1->llama-index-vector-stores-chroma) (3.3.2)\n",
            "Requirement already satisfied: greenlet!=0.4.17 in /usr/local/lib/python3.10/dist-packages (from SQLAlchemy[asyncio]>=1.4.49->llama-index-core<0.11.0,>=0.10.1->llama-index-vector-stores-chroma) (3.0.3)\n",
            "Requirement already satisfied: huggingface-hub<1.0,>=0.16.4 in /usr/local/lib/python3.10/dist-packages (from tokenizers>=0.13.2->chromadb<0.6.0,>=0.4.0->llama-index-vector-stores-chroma) (0.23.4)\n",
            "Requirement already satisfied: shellingham>=1.3.0 in /usr/local/lib/python3.10/dist-packages (from typer>=0.9.0->chromadb<0.6.0,>=0.4.0->llama-index-vector-stores-chroma) (1.5.4)\n",
            "Requirement already satisfied: rich>=10.11.0 in /usr/local/lib/python3.10/dist-packages (from typer>=0.9.0->chromadb<0.6.0,>=0.4.0->llama-index-vector-stores-chroma) (13.7.1)\n",
            "Requirement already satisfied: mypy-extensions>=0.3.0 in /usr/local/lib/python3.10/dist-packages (from typing-inspect>=0.8.0->llama-index-core<0.11.0,>=0.10.1->llama-index-vector-stores-chroma) (1.0.0)\n",
            "Requirement already satisfied: httptools>=0.5.0 in /usr/local/lib/python3.10/dist-packages (from uvicorn[standard]>=0.18.3->chromadb<0.6.0,>=0.4.0->llama-index-vector-stores-chroma) (0.6.1)\n",
            "Requirement already satisfied: python-dotenv>=0.13 in /usr/local/lib/python3.10/dist-packages (from uvicorn[standard]>=0.18.3->chromadb<0.6.0,>=0.4.0->llama-index-vector-stores-chroma) (1.0.1)\n",
            "Requirement already satisfied: uvloop!=0.15.0,!=0.15.1,>=0.14.0 in /usr/local/lib/python3.10/dist-packages (from uvicorn[standard]>=0.18.3->chromadb<0.6.0,>=0.4.0->llama-index-vector-stores-chroma) (0.19.0)\n",
            "Requirement already satisfied: watchfiles>=0.13 in /usr/local/lib/python3.10/dist-packages (from uvicorn[standard]>=0.18.3->chromadb<0.6.0,>=0.4.0->llama-index-vector-stores-chroma) (0.22.0)\n",
            "Requirement already satisfied: websockets>=10.4 in /usr/local/lib/python3.10/dist-packages (from uvicorn[standard]>=0.18.3->chromadb<0.6.0,>=0.4.0->llama-index-vector-stores-chroma) (12.0)\n",
            "Requirement already satisfied: marshmallow<4.0.0,>=3.18.0 in /usr/local/lib/python3.10/dist-packages (from dataclasses-json->llama-index-core<0.11.0,>=0.10.1->llama-index-vector-stores-chroma) (3.21.3)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas->llama-index-core<0.11.0,>=0.10.1->llama-index-vector-stores-chroma) (2023.4)\n",
            "Requirement already satisfied: tzdata>=2022.1 in /usr/local/lib/python3.10/dist-packages (from pandas->llama-index-core<0.11.0,>=0.10.1->llama-index-vector-stores-chroma) (2024.1)\n",
            "Requirement already satisfied: exceptiongroup in /usr/local/lib/python3.10/dist-packages (from anyio->httpx>=0.27.0->chromadb<0.6.0,>=0.4.0->llama-index-vector-stores-chroma) (1.2.1)\n",
            "Requirement already satisfied: dnspython>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from email_validator>=2.0.0->fastapi>=0.95.2->chromadb<0.6.0,>=0.4.0->llama-index-vector-stores-chroma) (2.6.1)\n",
            "Requirement already satisfied: cachetools<6.0,>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from google-auth>=1.0.1->kubernetes>=28.1.0->chromadb<0.6.0,>=0.4.0->llama-index-vector-stores-chroma) (5.3.3)\n",
            "Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.10/dist-packages (from google-auth>=1.0.1->kubernetes>=28.1.0->chromadb<0.6.0,>=0.4.0->llama-index-vector-stores-chroma) (0.4.0)\n",
            "Requirement already satisfied: rsa<5,>=3.1.4 in /usr/local/lib/python3.10/dist-packages (from google-auth>=1.0.1->kubernetes>=28.1.0->chromadb<0.6.0,>=0.4.0->llama-index-vector-stores-chroma) (4.9)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.16.4->tokenizers>=0.13.2->chromadb<0.6.0,>=0.4.0->llama-index-vector-stores-chroma) (3.15.4)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.10/dist-packages (from importlib-metadata<=7.1,>=6.0->opentelemetry-api>=1.2.0->chromadb<0.6.0,>=0.4.0->llama-index-vector-stores-chroma) (3.19.2)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2>=2.11.2->fastapi>=0.95.2->chromadb<0.6.0,>=0.4.0->llama-index-vector-stores-chroma) (2.1.5)\n",
            "Requirement already satisfied: markdown-it-py>=2.2.0 in /usr/local/lib/python3.10/dist-packages (from rich>=10.11.0->typer>=0.9.0->chromadb<0.6.0,>=0.4.0->llama-index-vector-stores-chroma) (3.0.0)\n",
            "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in /usr/local/lib/python3.10/dist-packages (from rich>=10.11.0->typer>=0.9.0->chromadb<0.6.0,>=0.4.0->llama-index-vector-stores-chroma) (2.16.1)\n",
            "Requirement already satisfied: humanfriendly>=9.1 in /usr/local/lib/python3.10/dist-packages (from coloredlogs->onnxruntime>=1.14.1->chromadb<0.6.0,>=0.4.0->llama-index-vector-stores-chroma) (10.0)\n",
            "Requirement already satisfied: mpmath<1.4,>=1.1.0 in /usr/local/lib/python3.10/dist-packages (from sympy->onnxruntime>=1.14.1->chromadb<0.6.0,>=0.4.0->llama-index-vector-stores-chroma) (1.3.0)\n",
            "Requirement already satisfied: mdurl~=0.1 in /usr/local/lib/python3.10/dist-packages (from markdown-it-py>=2.2.0->rich>=10.11.0->typer>=0.9.0->chromadb<0.6.0,>=0.4.0->llama-index-vector-stores-chroma) (0.1.2)\n",
            "Requirement already satisfied: pyasn1<0.7.0,>=0.4.6 in /usr/local/lib/python3.10/dist-packages (from pyasn1-modules>=0.2.1->google-auth>=1.0.1->kubernetes>=28.1.0->chromadb<0.6.0,>=0.4.0->llama-index-vector-stores-chroma) (0.6.0)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#!pip install 'elasticsearch<7.14.0'"
      ],
      "metadata": {
        "id": "r5FRgykT7TIx",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "f1285bfe-be6a-4016-d3a3-d6e202249a51"
      },
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: elasticsearch<7.14.0 in /usr/local/lib/python3.10/dist-packages (7.13.4)\n",
            "Requirement already satisfied: urllib3<2,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from elasticsearch<7.14.0) (1.26.19)\n",
            "Requirement already satisfied: certifi in /usr/local/lib/python3.10/dist-packages (from elasticsearch<7.14.0) (2024.7.4)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Импорт необходимых библиотек\n",
        "import os\n",
        "import time\n",
        "import json\n",
        "from elasticsearch import Elasticsearch, helpers\n",
        "from sentence_transformers import SentenceTransformer\n",
        "from transformers import AutoTokenizer, AutoModel\n",
        "from llama_index.core import Document\n",
        "from llama_index.core import Settings\n",
        "from llama_index.core import SimpleDirectoryReader\n",
        "from llama_index.core import StorageContext\n",
        "from llama_index.core import VectorStoreIndex"
      ],
      "metadata": {
        "id": "-Rrgn8RC7ZER",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "3f83a97c-fb5b-4080-ebf3-37a4f906af98"
      },
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/sentence_transformers/cross_encoder/CrossEncoder.py:11: TqdmExperimentalWarning: Using `tqdm.autonotebook.tqdm` in notebook mode. Use `tqdm.tqdm` instead to force console mode (e.g. in jupyter console)\n",
            "  from tqdm.autonotebook import tqdm, trange\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Импорт библиотек TensorFlow и TensorFlow I/O\n",
        "import tensorflow as tf\n",
        "import tensorflow_io as tfio\n",
        "\n",
        "# Импорт дополнительных модулей из TensorFlow\n",
        "from tensorflow.keras import layers\n",
        "from tensorflow.keras.layers.experimental import preprocessing"
      ],
      "metadata": {
        "id": "GjYcOLe__w8C"
      },
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Скачивание и распаковка Elasticsearch\n",
        "%%bash\n",
        "\n",
        "wget -q https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-oss-7.10.0-linux-x86_64.tar.gz\n",
        "wget -q https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-oss-7.10.0-linux-x86_64.tar.gz.sha512\n",
        "tar -xzf elasticsearch-oss-7.10.0-linux-x86_64.tar.gz\n",
        "sudo chown -R daemon:daemon elasticsearch-7.10.0/\n",
        "shasum -a 512 -c elasticsearch-oss-7.10.0-linux-x86_64.tar.gz.sha512"
      ],
      "metadata": {
        "id": "EAPKRNM3AGIp",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "8a3fa08c-e465-4f43-b24e-c111d88a4c5c"
      },
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "elasticsearch-oss-7.10.0-linux-x86_64.tar.gz: OK\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "id": "AXBegrGNc7j_"
      },
      "outputs": [],
      "source": [
        "# Запуск Elasticsearch в фоновом режиме\n",
        "%%bash --bg\n",
        "\n",
        "sudo -H -u daemon elasticsearch-7.10.0/bin/elasticsearch"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {
        "id": "kiNRKVHNc-2g"
      },
      "outputs": [],
      "source": [
        "import time\n",
        "time.sleep(30)"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Проверка запущенных процессов Elasticsearch\n",
        "%%bash\n",
        "\n",
        "ps -ef | grep elasticsearch"
      ],
      "metadata": {
        "id": "oqGkSezR_xEr",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "3f85eb8f-d00f-4423-c068-48b9511dbb77"
      },
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "root       33719   33717  0 06:28 ?        00:00:00 sudo -H -u daemon elasticsearch-7.10.0/bin/elast\n",
            "daemon     33720   33719 78 06:28 ?        00:00:23 /content/elasticsearch-7.10.0/jdk/bin/java -Xsha\n",
            "root       34038   34036  0 06:29 ?        00:00:00 grep elasticsearch\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Проверка соединения с Elasticsearch\n",
        "%%bash\n",
        "\n",
        "curl -sX GET \"localhost:9200/\""
      ],
      "metadata": {
        "id": "E3iLr26dASG6",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "38c085e2-697b-4106-d553-678d18a7c5bd"
      },
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "{\n",
            "  \"name\" : \"9dda0200f192\",\n",
            "  \"cluster_name\" : \"elasticsearch\",\n",
            "  \"cluster_uuid\" : \"RGWmq7oITz2ZwIdygJuRLw\",\n",
            "  \"version\" : {\n",
            "    \"number\" : \"7.10.0\",\n",
            "    \"build_flavor\" : \"oss\",\n",
            "    \"build_type\" : \"tar\",\n",
            "    \"build_hash\" : \"51e9d6f22758d0374a0f3f5c6e8f3a7997850f96\",\n",
            "    \"build_date\" : \"2020-11-09T21:30:33.964949Z\",\n",
            "    \"build_snapshot\" : false,\n",
            "    \"lucene_version\" : \"8.7.0\",\n",
            "    \"minimum_wire_compatibility_version\" : \"6.8.0\",\n",
            "    \"minimum_index_compatibility_version\" : \"6.0.0-beta1\"\n",
            "  },\n",
            "  \"tagline\" : \"You Know, for Search\"\n",
            "}\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Создание объекта Elasticsearch, подключенный к локальному экземпляру Elasticsearch,\n",
        "# который запущен на порту 9200 по протоколу HTTP\n",
        "es = Elasticsearch(hosts=[{\"host\": \"localhost\", \"port\": 9200, \"scheme\": \"http\"}])\n",
        "es.ping()"
      ],
      "metadata": {
        "id": "Luze09xMASKx",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "baf8b5e0-f9d4-4f43-a92d-946127981e4e"
      },
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {},
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Загрузка модели для векторных представлений\n",
        "embed_model = SentenceTransformer('distiluse-base-multilingual-cased')\n",
        "# Загрузка модели для генерации ответов\n",
        "model = 'IlyaGusev/saiga_mistral_7b_lora'"
      ],
      "metadata": {
        "id": "grPiDqc97kS-"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Функция для генерации ответов\n",
        "def generate_response(question):\n",
        "    inputs = tokenizer.encode(question, return_tensors=\"pt\")\n",
        "    outputs = model.generate(inputs, max_length=150, num_return_sequences=1)\n",
        "    response = tokenizer.decode(outputs[0], skip_special_tokens=True)\n",
        "    return response"
      ],
      "metadata": {
        "id": "Q6wWv-cE7wF3"
      },
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Загрузка и индексация статей\n",
        "def index_articles(articles_path):\n",
        "    with open(articles_path, 'r', encoding='utf-8') as file:\n",
        "        articles = json.load(file)['pages']\n",
        "\n",
        "    # Создание индекса в Elasticsearch, если он не существует\n",
        "    if not es.indices.exists(index=\"articles\"):\n",
        "        es_index = {\n",
        "            \"settings\": {\n",
        "                \"index\": {\n",
        "                    \"number_of_shards\": 1,\n",
        "                    \"number_of_replicas\": 0\n",
        "                }\n",
        "            },\n",
        "            \"mappings\": {\n",
        "                \"properties\": {\n",
        "                    \"title\": {\"type\": \"text\"},\n",
        "                    \"text\": {\"type\": \"text\"},\n",
        "                    \"link\": {\"type\": \"keyword\"},\n",
        "                    \"text_vector\": {\"type\": \"dense_vector\", \"dims\": 512}\n",
        "                }\n",
        "            }\n",
        "        }\n",
        "        es.indices.create(index=\"articles\", body=es_index)\n",
        "\n",
        "    # Индексация статей в Elasticsearch\n",
        "    for article in articles:\n",
        "        text_vector = embed_model.encode(article['text'])\n",
        "        doc = {\n",
        "            \"title\": article['title'],\n",
        "            \"text\": article['text'],\n",
        "            \"link\": article['link'],\n",
        "            \"text_vector\": text_vector\n",
        "        }\n",
        "        es.index(index=\"articles\", body=doc)"
      ],
      "metadata": {
        "id": "YVeZ-Hqz77CP"
      },
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "LI78wXhr541N"
      },
      "outputs": [],
      "source": [
        "# Функция для интерактивного поиска и генерации ответов\n",
        "def interactive_search():\n",
        "    while True:\n",
        "        inp_question = input(\"Пожалуйста, введите вопрос: \")\n",
        "\n",
        "        # Вычисление векторного представления вопроса\n",
        "        encode_start_time = time.time()\n",
        "        question_embedding = embed_model.encode(inp_question)\n",
        "        encode_end_time = time.time()\n",
        "\n",
        "        # Лексический поиск с использованием TF-IDF\n",
        "        tfidf_search = es.search(index=\"articles\", body={\n",
        "            \"query\": {\n",
        "                \"multi_match\": {\n",
        "                    \"query\": inp_question,\n",
        "                    \"fields\": [\"title\", \"text\"],\n",
        "                    \"type\": \"best_fields\"\n",
        "                }\n",
        "            }\n",
        "        })\n",
        "\n",
        "        # Семантический поиск с использованием векторных представлений\n",
        "        sem_search = es.search(\n",
        "            index=\"articles\",\n",
        "            body={\n",
        "                \"query\": {\n",
        "                    \"script_score\": {\n",
        "                        \"query\": {\"match_all\": {}},\n",
        "                        \"script\": {\n",
        "                            \"source\": \"cosineSimilarity(params.query_vector, 'text_vector') + 1.0\",\n",
        "                            \"params\": {\"query_vector\": question_embedding.tolist()}\n",
        "                        }\n",
        "                    }\n",
        "                }\n",
        "            }\n",
        "        )\n",
        "\n",
        "        # Вывод результатов поиска и времени выполнения\n",
        "        print(\"Введенный вопрос:\", inp_question)\n",
        "        print(\n",
        "            \"Вычисление эмбеддинга заняло {:.3f} секунд, поиск по TF-IDF занял {:.3f} секунд, семантический поиск с ES занял {:.3f} секунд\".format(\n",
        "                encode_end_time - encode_start_time, tfidf_search[\"took\"] / 1000, sem_search[\"took\"] / 1000\n",
        "            )\n",
        "        )\n",
        "\n",
        "        print(\"Результаты поиска по TF-IDF:\")\n",
        "        for hit in tfidf_search[\"hits\"][\"hits\"][0:5]:\n",
        "            print(\"\\t{}\".format(hit[\"_source\"][\"text\"]))\n",
        "            print(\"\\tСсылка: {}\".format(hit[\"_source\"][\"link\"]))\n",
        "\n",
        "        print(\"\\nРезультаты семантического поиска:\")\n",
        "        for hit in sem_search[\"hits\"][\"hits\"][0:5]:\n",
        "            print(\"\\t{}\".format(hit[\"_source\"][\"text\"]))\n",
        "            print(\"\\tСсылка: {}\".format(hit[\"_source\"][\"link\"]))\n",
        "\n",
        "        # Генерация ответа на вопрос\n",
        "        response = generate_response(inp_question)\n",
        "        print(\"\\nОтвет от модели SaigaMistral:\")\n",
        "        print(response)\n",
        "\n",
        "        print(\"\\n\\n========\\n\")\n",
        "\n",
        "# Основная функция\n",
        "if __name__ == \"__main__\":\n",
        "    articles_path = \"/content/ConfluencePages.json\"\n",
        "    index_articles(articles_path)\n",
        "    interactive_search()"
      ]
    }
  ]
}