function_calling.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Import Python Libraries and Set Up the Environment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "import base64\n",
    "import json\n",
    "import getpass\n",
    "from openai import AzureOpenAI\n",
    "\n",
    "client = AzureOpenAI(\n",
    "    # api_key=os.getenv(\"AZURE_OPENAI_API_KEY\"),\n",
    "    api_key=getpass.getpass(\"Enter your Azure OpenAI API key: \"),\n",
    "    api_version=\"2024-07-01-preview\",\n",
    "    # azure_endpoint=os.getenv(\"AZURE_OPENAI_ENDPOINT\"),\n",
    "    azure_endpoint=getpass.getpass(\"Enter your Azure OpenAI endpoint: \")\n",
    ")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Write and define your functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_image() -> bytes:\n",
    "    response = requests.get(\"http://localhost:30080/deviceshifu-camera/capture\")\n",
    "    return response.content\n",
    "\n",
    "def set_plc_output(switch: bool) -> str:\n",
    "    output = 0\n",
    "    if switch:\n",
    "        output = 1\n",
    "    response = requests.get(\"http://localhost:30080/deviceshifu-plc/sendsinglebit?rootaddress=Q&address=0&start=0&digit=0&value=\" + str(output))\n",
    "    response = requests.get(\"http://localhost:30080/deviceshifu-plc/sendsinglebit?rootaddress=Q&address=0&start=0&digit=1&value=\" + str(output))\n",
    "    response = requests.get(\"http://localhost:30080/deviceshifu-plc/sendsinglebit?rootaddress=Q&address=0&start=0&digit=2&value=\" + str(output))\n",
    "    response = requests.get(\"http://localhost:30080/deviceshifu-plc/sendsinglebit?rootaddress=Q&address=0&start=0&digit=3&value=\" + str(output))\n",
    "    return response.content\n",
    "\n",
    "def set_led(num: int) -> str:\n",
    "    response = requests.post(\"http://localhost:30080/deviceshifu-led/number\", json={\"value\": num})\n",
    "    return response.content\n",
    "\n",
    "tools = [\n",
    "    {\n",
    "        \"type\": \"function\",\n",
    "        \"function\": {\n",
    "            \"name\": \"get_image\",\n",
    "            \"description\": \"A function that gets an image from the /capture endpoint.\",\n",
    "        },\n",
    "    },\n",
    "    {\n",
    "        \"type\": \"function\",\n",
    "        \"function\": {\n",
    "            \"name\": \"set_plc_output\",\n",
    "            \"description\": \"A function that sets the output of the PLC.\",\n",
    "            \"parameters\": {\n",
    "                \"type\": \"object\",\n",
    "                \"properties\": {\n",
    "                    \"switch\": {\n",
    "                        \"type\": \"boolean\",\n",
    "                        \"description\": \"To set the output to True or False\",\n",
    "                    },\n",
    "                },\n",
    "                \"required\": [\"switch\"],\n",
    "                \"additionalProperties\": False,\n",
    "            }\n",
    "        },\n",
    "    },\n",
    "    {\n",
    "        \"type\": \"function\",\n",
    "        \"function\": {\n",
    "            \"name\": \"set_led\",\n",
    "            \"description\": \"A function that sets the display number of the LED.\",\n",
    "            \"parameters\": {\n",
    "                \"type\": \"object\",\n",
    "                \"properties\": {\n",
    "                    \"num\": {\n",
    "                        \"type\": \"integer\",\n",
    "                        \"description\": \"a number to display on the LED, between 0 and 9999\",\n",
    "                    },\n",
    "                },\n",
    "                \"required\": [\"num\"],\n",
    "                \"additionalProperties\": False,\n",
    "            }\n",
    "        },\n",
    "    },\n",
    "]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Function call handler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "metadata": {},
   "outputs": [],
   "source": [
    "def handle_function_call(tool_call):\n",
    "    if tool_call.function.name == \"get_image\":\n",
    "        image = get_image()\n",
    "        with open(\"image.png\", \"wb\") as f:\n",
    "            f.write(image)\n",
    "        return base64.b64encode(image).decode('utf-8')\n",
    "    elif tool_call.function.name == \"set_plc_output\":\n",
    "        arguments = json.loads(tool_call.function.arguments)\n",
    "        switch = arguments['switch']\n",
    "        return set_plc_output(switch)\n",
    "    elif tool_call.function.name == \"set_led\":\n",
    "        arguments = json.loads(tool_call.function.arguments)\n",
    "        num = arguments['num']\n",
    "        return set_led(num)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1. Call GPT to take a picture and describe what's in it"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The display shows the number **3999**. As for the PLC, it appears that **only one light is on**; the others are not illuminated.\n"
     ]
    }
   ],
   "source": [
    "prompt = [\n",
    "    {\n",
    "        \"role\": \"system\",\n",
    "        \"content\": \"You are a helpful embodied intelligence assistant. Use the supplied tools to assist the user.\",\n",
    "    },\n",
    "    {   \"role\": \"user\",\n",
    "        \"content\": \"Hi, can you tell what the camera is seeing?\"\n",
    "    },\n",
    "]\n",
    "\n",
    "chat_completion = client.chat.completions.create(\n",
    "    model=\"gpt-4o-mini\", messages=prompt, tools=tools\n",
    ")\n",
    "tool_call = chat_completion.choices[0].message.tool_calls[0]\n",
    "function_call_output = handle_function_call(tool_call)\n",
    "\n",
    "prompt.append(chat_completion.choices[0].message)\n",
    "prompt.append(\n",
    "    {\n",
    "          \"tool_call_id\": tool_call.id,\n",
    "          \"role\": \"tool\",\n",
    "          \"name\": tool_call.function.name,\n",
    "          \"content\": \"success\",\n",
    "    }\n",
    ")\n",
    "prompt.append(\n",
    "    {\n",
    "        \"role\": \"user\",\n",
    "        \"content\": [\n",
    "            {\n",
    "                \"type\": \"text\",\n",
    "                \"text\": \"Thanks! Can you tell me what is in the image? Specifically what is the number on the display and does the PLC has 4 output lights on?\",\n",
    "            },\n",
    "            {\n",
    "                \"type\": \"image_url\",\n",
    "                \"image_url\": {\n",
    "                    \"url\": f\"data:image/jpeg;base64,{function_call_output}\",\n",
    "                },\n",
    "            },\n",
    "        ],\n",
    "    }\n",
    ")\n",
    "chat_completion = client.chat.completions.create(\n",
    "    model=\"gpt-4o-mini\", messages=prompt, tools=tools\n",
    ")\n",
    "print(chat_completion.choices[0].message.content)\n",
    "# remove last prompt\n",
    "prompt.pop()\n",
    "prompt.append(\n",
    "    {\n",
    "        \"role\": \"user\",\n",
    "        \"content\": \"Thanks! Can you tell me what is in the image? Specifically what is the number on the display and does the PLC has 4 output lights on?\"\n",
    "    },\n",
    ")\n",
    "# append the response to the prompt\n",
    "prompt.append(chat_completion.choices[0].message)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2. Increase the LED display number by 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ChatCompletionMessageToolCall(id='call_BK3bSIH9RwXjnicknZJw6TFS', function=Function(arguments='{\"num\":4004}', name='set_led'), type='function')\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "# ask the model to increase the display number on the LED by 5\n",
    "prompt.append(\n",
    "    {\"role\": \"user\", \"content\": \"Can you increase the display number on the LED by 5?\"}\n",
    ")\n",
    "\n",
    "\n",
    "\n",
    "chat_completion = client.chat.completions.create(\n",
    "    model=\"gpt-4o-mini\", messages=prompt, tools=tools\n",
    ")\n",
    "tool_call = chat_completion.choices[0].message.tool_calls[0]\n",
    "print(tool_call)\n",
    "function_call_output = handle_function_call(tool_call)\n",
    "print(chat_completion.choices[0].message.content)\n",
    "# append the response to the prompt\n",
    "\n",
    "prompt.append(chat_completion.choices[0].message)\n",
    "prompt.append(\n",
    "    {\n",
    "          \"tool_call_id\": tool_call.id,\n",
    "          \"role\": \"tool\",\n",
    "          \"name\": tool_call.function.name,\n",
    "          \"content\": \"success\",\n",
    "    }\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3. Set the PLC output to true if it's off and false if it's on"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ChatCompletionMessageToolCall(id='call_BK3bSIH9RwXjnicknZJw6TFS', function=Function(arguments='{\"switch\":true}', name='set_plc_output'), type='function')\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "# ask the model to turn on the PLC output if it's off and turn it off if it's on\n",
    "prompt.append(\n",
    "    {\"role\": \"user\", \"content\": \"Can you set the PLC output to true if it doesn't have 4 output lights on and off if it does?\"}\n",
    ")\n",
    "\n",
    "\n",
    "\n",
    "chat_completion = client.chat.completions.create(\n",
    "    model=\"gpt-4o-mini\", messages=prompt, tools=tools\n",
    ")\n",
    "tool_call = chat_completion.choices[0].message.tool_calls[0]\n",
    "print(tool_call)\n",
    "function_call_output = handle_function_call(tool_call)\n",
    "print(chat_completion.choices[0].message.content)\n",
    "# append the response to the prompt\n",
    "prompt.append(chat_completion.choices[0].message)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(prompt)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "### Thank you!\n",
    "\n",
    "Any questions or feedback? Please let me know!\n",
    "\n",
    "Email: xqin@edgenesis.com"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}