Skip to content

Commit 918021d

Browse files
StephenHodgsonStephenHodgson
authored
com.openai.unity 8.6.3 (#349)
- Fixed Threads.AssistantResponse serialization with reasoning_effort - Updated AssitantBehaviour sample - Updated ChatBehaviour sample - Added additional deserializing json logging when a failure occurs
1 parent b622276 commit 918021d

File tree

7 files changed

+113
-76
lines changed

7 files changed

+113
-76
lines changed

‎.gitignore

+1
Original file line numberDiff line numberDiff line change
@@ -88,3 +88,4 @@ OpenAI/Assets/Plugins.meta
8888
OpenAI/Assets/Resources/
8989
OpenAI/Assets/Resources.meta
9090
OpenAI/ProjectSettings/SceneTemplateSettings.json
91+
OpenAI/ProjectSettings/boot.config

‎OpenAI/Packages/com.openai.unity/Runtime/Assistants/AssistantResponse.cs

+7-5
Original file line numberDiff line numberDiff line change
@@ -27,8 +27,9 @@ internal AssistantResponse(
2727
[JsonProperty("tools")] IReadOnlyList<Tool> tools,
2828
[JsonProperty("tool_resources")] ToolResources toolResources,
2929
[JsonProperty("metadata")] Dictionary<string, string> metadata,
30-
[JsonProperty("temperature")] double temperature,
31-
[JsonProperty("top_p")] double topP,
30+
[JsonProperty("temperature")] float? temperature,
31+
[JsonProperty("top_p")] float? topP,
32+
[JsonProperty("reasoning_effort")] ReasoningEffort? reasoningEffort,
3233
[JsonProperty("response_format")][JsonConverter(typeof(ResponseFormatConverter))] ResponseFormatObject responseFormat)
3334
{
3435
Id = id;
@@ -43,6 +44,7 @@ internal AssistantResponse(
4344
Metadata = metadata;
4445
Temperature = temperature;
4546
TopP = topP;
47+
ReasoningEffort = reasoningEffort;
4648
ResponseFormatObject = responseFormat;
4749
}
4850

@@ -139,7 +141,7 @@ internal AssistantResponse(
139141
/// </summary>
140142
[Preserve]
141143
[JsonProperty("temperature", DefaultValueHandling = DefaultValueHandling.Ignore)]
142-
public double Temperature { get; }
144+
public float? Temperature { get; }
143145

144146
/// <summary>
145147
/// An alternative to sampling with temperature, called nucleus sampling,
@@ -148,7 +150,7 @@ internal AssistantResponse(
148150
/// </summary>
149151
[Preserve]
150152
[JsonProperty("top_p", DefaultValueHandling = DefaultValueHandling.Ignore)]
151-
public double TopP { get; }
153+
public float? TopP { get; }
152154

153155
/// <summary>
154156
/// Constrains effort on reasoning for reasoning models.
@@ -157,7 +159,7 @@ internal AssistantResponse(
157159
/// </summary>
158160
[Preserve]
159161
[JsonProperty("reasoning_effort", DefaultValueHandling = DefaultValueHandling.Ignore)]
160-
public ReasoningEffort ReasoningEffort { get; }
162+
public ReasoningEffort? ReasoningEffort { get; }
161163

162164
/// <summary>
163165
/// Specifies the format that the model must output.

‎OpenAI/Packages/com.openai.unity/Runtime/Extensions/ResponseExtensions.cs

+12-3
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
using Newtonsoft.Json;
44
using System;
55
using System.Globalization;
6+
using UnityEngine;
67
using Utilities.WebRequestRest;
78

89
namespace OpenAI.Extensions
@@ -99,9 +100,17 @@ internal static void SetResponseData(this BaseResponse response, Response restRe
99100

100101
internal static T Deserialize<T>(this Response response, OpenAIClient client) where T : BaseResponse
101102
{
102-
var result = JsonConvert.DeserializeObject<T>(response.Body, OpenAIClient.JsonSerializationOptions);
103-
result.SetResponseData(response, client);
104-
return result;
103+
try
104+
{
105+
var result = JsonConvert.DeserializeObject<T>(response.Body, OpenAIClient.JsonSerializationOptions);
106+
result.SetResponseData(response, client);
107+
return result;
108+
}
109+
catch (Exception)
110+
{
111+
Debug.LogError($"Failed to deserialize:\n{response.Body}");
112+
throw;
113+
}
105114
}
106115
}
107116
}

‎OpenAI/Packages/com.openai.unity/Samples~/Assistant/AssistantBehaviour.cs

+87-62
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,6 @@
77
using OpenAI.Threads;
88
using System;
99
using System.Collections.Generic;
10-
using System.Diagnostics;
1110
using System.Linq;
1211
using System.Threading;
1312
using System.Threading.Tasks;
@@ -104,12 +103,8 @@ private async void Awake()
104103
{
105104
Tool.GetOrCreateTool(openAI.ImagesEndPoint, nameof(ImagesEndpoint.GenerateImageAsync))
106105
}),
107-
destroyCancellationToken);
108-
109-
thread = await openAI.ThreadsEndpoint.CreateThreadAsync(
110-
new CreateThreadRequest(assistant),
111-
destroyCancellationToken);
112-
106+
destroyCancellationToken);
107+
thread = await openAI.ThreadsEndpoint.CreateThreadAsync(new(assistant), destroyCancellationToken);
113108
inputField.onSubmit.AddListener(SubmitChat);
114109
submitButton.onClick.AddListener(SubmitChat);
115110
recordButton.onClick.AddListener(ToggleRecording);
@@ -123,6 +118,8 @@ private async void Awake()
123118
{
124119
switch (e)
125120
{
121+
case TaskCanceledException:
122+
case OperationCanceledException:
126123
case ObjectDisposedException:
127124
// ignored
128125
break;
@@ -158,7 +155,15 @@ private async void Awake()
158155
}
159156
catch (Exception e)
160157
{
161-
Debug.LogError(e);
158+
switch (e)
159+
{
160+
case TaskCanceledException:
161+
case OperationCanceledException:
162+
break;
163+
default:
164+
Debug.LogError(e);
165+
break;
166+
}
162167
}
163168
}
164169
}
@@ -182,6 +187,7 @@ private async void SubmitChat()
182187
inputField.ReleaseSelection();
183188
inputField.interactable = false;
184189
submitButton.interactable = false;
190+
recordButton.interactable = false;
185191
var userMessage = new Message(inputField.text);
186192
var userMessageContent = AddNewTextMessageContent(Role.User);
187193
userMessageContent.text = $"User: {inputField.text}";
@@ -192,6 +198,57 @@ private async void SubmitChat()
192198
try
193199
{
194200
await thread.CreateMessageAsync(userMessage, destroyCancellationToken);
201+
202+
async Task StreamEventHandler(IServerSentEvent @event)
203+
{
204+
try
205+
{
206+
switch (@event)
207+
{
208+
case MessageResponse message:
209+
switch (message.Status)
210+
{
211+
case MessageStatus.InProgress:
212+
if (message.Role == Role.Assistant)
213+
{
214+
assistantMessageContent.text += message.PrintContent();
215+
scrollView.verticalNormalizedPosition = 0f;
216+
}
217+
break;
218+
case MessageStatus.Completed:
219+
if (message.Role == Role.Assistant)
220+
{
221+
await GenerateSpeechAsync(message.PrintContent(), destroyCancellationToken);
222+
scrollView.verticalNormalizedPosition = 0f;
223+
}
224+
break;
225+
}
226+
break;
227+
case RunResponse run:
228+
if (run.Status == RunStatus.RequiresAction)
229+
{
230+
await ProcessToolCalls(run);
231+
}
232+
233+
break;
234+
case Error errorResponse:
235+
throw errorResponse.Exception ?? new Exception(errorResponse.Message);
236+
}
237+
}
238+
catch (Exception e)
239+
{
240+
switch (e)
241+
{
242+
case TaskCanceledException:
243+
case OperationCanceledException:
244+
break;
245+
default:
246+
Debug.LogError(e);
247+
break;
248+
}
249+
}
250+
}
251+
195252
var run = await thread.CreateRunAsync(assistant, StreamEventHandler, destroyCancellationToken);
196253
await run.WaitForStatusChangeAsync(timeout: 60, cancellationToken: destroyCancellationToken);
197254
}
@@ -212,55 +269,14 @@ private async void SubmitChat()
212269
if (destroyCancellationToken is { IsCancellationRequested: false })
213270
{
214271
inputField.interactable = true;
215-
EventSystem.current.SetSelectedGameObject(inputField.gameObject);
216272
submitButton.interactable = true;
273+
recordButton.interactable = true;
274+
EventSystem.current.SetSelectedGameObject(inputField.gameObject);
217275
}
218276

219277
isChatPending = false;
220278
}
221279

222-
async Task StreamEventHandler(IServerSentEvent @event)
223-
{
224-
try
225-
{
226-
switch (@event)
227-
{
228-
case MessageResponse message:
229-
switch (message.Status)
230-
{
231-
case MessageStatus.InProgress:
232-
if (message.Role == Role.Assistant)
233-
{
234-
assistantMessageContent.text += message.PrintContent();
235-
scrollView.verticalNormalizedPosition = 0f;
236-
}
237-
break;
238-
case MessageStatus.Completed:
239-
if (message.Role == Role.Assistant)
240-
{
241-
await GenerateSpeechAsync(message.PrintContent(), destroyCancellationToken);
242-
}
243-
break;
244-
}
245-
break;
246-
case RunResponse run:
247-
switch (run.Status)
248-
{
249-
case RunStatus.RequiresAction:
250-
await ProcessToolCalls(run);
251-
break;
252-
}
253-
break;
254-
case Error errorResponse:
255-
throw errorResponse.Exception ?? new Exception(errorResponse.Message);
256-
}
257-
}
258-
catch (Exception e)
259-
{
260-
Debug.LogError(e);
261-
}
262-
}
263-
264280
async Task ProcessToolCalls(RunResponse run)
265281
{
266282
Debug.Log(nameof(ProcessToolCalls));
@@ -315,20 +331,17 @@ private async Task GenerateSpeechAsync(string text, CancellationToken cancellati
315331
#pragma warning disable CS0612 // Type or member is obsolete
316332
var request = new SpeechRequest(text, Model.TTS_1, voice, SpeechResponseFormat.PCM);
317333
#pragma warning restore CS0612 // Type or member is obsolete
318-
var stopwatch = Stopwatch.StartNew();
319334
var speechClip = await openAI.AudioEndpoint.GetSpeechAsync(request, partialClip =>
320335
{
321336
streamAudioSource.BufferCallback(partialClip.AudioSamples);
322337
}, cancellationToken);
323-
var playbackTime = speechClip.AudioClip.length - (float)stopwatch.Elapsed.TotalSeconds + 0.1f;
338+
await Awaiters.DelayAsync(TimeSpan.FromSeconds(speechClip.Length), cancellationToken).ConfigureAwait(true);
339+
((AudioSource)streamAudioSource).clip = speechClip.AudioClip;
324340

325341
if (enableDebug)
326342
{
327343
Debug.Log(speechClip.CachePath);
328344
}
329-
330-
await Awaiters.DelayAsync(TimeSpan.FromSeconds(playbackTime), cancellationToken).ConfigureAwait(true);
331-
((AudioSource)streamAudioSource).clip = speechClip.AudioClip;
332345
}
333346
finally
334347
{
@@ -391,7 +404,9 @@ private async void ProcessRecording(Tuple<string, AudioClip> recording)
391404

392405
try
393406
{
407+
inputField.interactable = false;
394408
recordButton.interactable = false;
409+
submitButton.interactable = false;
395410
var request = new AudioTranscriptionRequest(clip, temperature: 0.1f, language: "en");
396411
var userInput = await openAI.AudioEndpoint.CreateTranscriptionTextAsync(request, destroyCancellationToken);
397412

@@ -405,12 +420,22 @@ private async void ProcessRecording(Tuple<string, AudioClip> recording)
405420
}
406421
catch (Exception e)
407422
{
408-
Debug.LogError(e);
409-
inputField.interactable = true;
410-
}
411-
finally
412-
{
413-
recordButton.interactable = true;
423+
if (!isChatPending)
424+
{
425+
inputField.interactable = true;
426+
recordButton.interactable = true;
427+
submitButton.interactable = true;
428+
}
429+
430+
switch (e)
431+
{
432+
case TaskCanceledException:
433+
case OperationCanceledException:
434+
break;
435+
default:
436+
Debug.LogError(e);
437+
break;
438+
}
414439
}
415440
}
416441
}

‎OpenAI/Packages/com.openai.unity/Samples~/Chat/ChatBehaviour.cs

+4-4
Original file line numberDiff line numberDiff line change
@@ -250,15 +250,15 @@ private async Task GenerateSpeechAsync(string text, CancellationToken cancellati
250250
{
251251
streamAudioSource.BufferCallback(partialClip.AudioSamples);
252252
}, cancellationToken);
253-
var playbackTime = speechClip.AudioClip.length - (float)stopwatch.Elapsed.TotalSeconds + 0.1f;
253+
var playbackTime = speechClip.Length - (float)stopwatch.Elapsed.TotalSeconds + 0.1f;
254+
255+
await Awaiters.DelayAsync(TimeSpan.FromSeconds(playbackTime), cancellationToken).ConfigureAwait(true);
256+
((AudioSource)streamAudioSource).clip = speechClip.AudioClip;
254257

255258
if (enableDebug)
256259
{
257260
Debug.Log(speechClip.CachePath);
258261
}
259-
260-
await Awaiters.DelayAsync(TimeSpan.FromSeconds(playbackTime), cancellationToken).ConfigureAwait(true);
261-
((AudioSource)streamAudioSource).clip = speechClip.AudioClip;
262262
}
263263

264264
private TextMeshProUGUI AddNewTextMessageContent(Role role)

‎OpenAI/Packages/com.openai.unity/package.json

+1-1
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
"displayName": "OpenAI",
44
"description": "A OpenAI package for the Unity Game Engine to use GPT-4, GPT-3.5, GPT-3 and Dall-E though their RESTful API (currently in beta).\n\nIndependently developed, this is not an official library and I am not affiliated with OpenAI.\n\nAn OpenAI API account is required.",
55
"keywords": [],
6-
"version": "8.6.2",
6+
"version": "8.6.3",
77
"unity": "2021.3",
88
"documentationUrl": "https://github.com/RageAgainstThePixel/com.openai.unity#documentation",
99
"changelogUrl": "https://github.com/RageAgainstThePixel/com.openai.unity/releases",

‎OpenAI/Packages/manifest.json

+1-1
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
"com.unity.ide.visualstudio": "2.0.22",
55
"com.unity.textmeshpro": "3.0.9",
66
"com.unity.ugui": "1.0.0",
7-
"com.utilities.buildpipeline": "1.6.0"
7+
"com.utilities.buildpipeline": "1.6.1"
88
},
99
"scopedRegistries": [
1010
{

0 commit comments

Comments
 (0)