Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Vertex AI] Test SDK with v1 API instead of v1beta #14345

Draft
wants to merge 2 commits into
base: main
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion FirebaseVertexAI/Sources/GenerativeAIRequest.swift
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ public struct RequestOptions {
let timeout: TimeInterval

/// The API version to use in requests to the backend.
let apiVersion = "v1beta"
let apiVersion = "v1"

/// Initializes a request options object.
///
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@

// MARK: - Generate Content

func testGenerateContent() async throws {
func testGenerateContent_text() async throws {
let prompt = "Where is Google headquarters located? Answer with the city name only."

let response = try await model.generateContent(prompt)
Expand All @@ -75,6 +75,31 @@
XCTAssertEqual(text, "Mountain View")
}

func testGenerateContent_image_fileData_public() async throws {
let storageRef = storage.reference(withPath: "vertexai/public/green.png")
let fileData = FileDataPart(uri: storageRef.gsURI, mimeType: "image/png")

let response = try await model.generateContent(fileData, "What color is this?")

XCTAssertNotNil(response.text)
}

func testGenerateContent_image_fileData_requiresUserAuth_wrongUser_permissionDenied() async throws {
let userID = "3MjEzU6JIobWvHdCYHicnDMcPpQ2"
let storageRef = storage.reference(withPath: "vertexai/authenticated/user/\(userID)/pink.webp")

let fileData = FileDataPart(uri: storageRef.gsURI, mimeType: "image/webp")

do {
let response = try await model.generateContent(fileData, "What color is this?")
XCTFail("Expected to throw an error, got response: \(response)")

Check failure on line 95 in FirebaseVertexAI/Tests/TestApp/Tests/Integration/IntegrationTests.swift

View workflow job for this annotation

GitHub Actions / testapp-integration (iOS, macos-15)

testGenerateContent_image_fileData_requiresUserAuth_wrongUser_permissionDenied, failed - Expected to throw an error, got response: GenerateContentResponse(candidates: [FirebaseVertexAI.Candidate(content: FirebaseVertexAI.ModelContent(role: Optional("model"), internalParts: [FirebaseVertexAI.ModelContent.InternalPart.text("The color is pink. \n")]), safetyRatings: [FirebaseVertexAI.SafetyRating(category: FirebaseVertexAI.HarmCategory(rawValue: "HARM_CATEGORY_HATE_SPEECH"), probability: FirebaseVertexAI.SafetyRating.HarmProbability(rawValue: "NEGLIGIBLE"), probabilityScore: 0.05493164, severity: FirebaseVertexAI.SafetyRating.HarmSeverity(rawValue: "HARM_SEVERITY_NEGLIGIBLE"), severityScore: 0.1953125, blocked: false), FirebaseVertexAI.SafetyRating(category: FirebaseVertexAI.HarmCategory(rawValue: "HARM_CATEGORY_DANGEROUS_CONTENT"), probability: FirebaseVertexAI.SafetyRating.HarmProbability(rawValue: "NEGLIGIBLE"), probabilityScore: 0.056640625, severity: FirebaseVertexAI.SafetyRating.HarmSeverity(rawValue: "HARM_SEVERITY_NEGLIGIBLE"), severityScore: 0.08154297, blocked: false), FirebaseVertexAI.SafetyRating(category: FirebaseVertexAI.HarmCategory(rawValue: "HARM_CATEGORY_HARASSMENT"), probability: FirebaseVertexAI.SafetyRating.HarmProbability(rawValue: "NEGLIGIBLE"), probabilityScore: 0.057373047, severity: FirebaseVertexAI.SafetyRating.HarmSeverity(rawValue: "HARM_SEVERITY_NEGLIGIBLE"), severityScore: 0.075683594, blocked: false), FirebaseVertexAI.SafetyRating(category: FirebaseVertexAI.HarmCategory(rawValue: "HARM_CATEGORY_SEXUALLY_EXPLICIT"), probability: FirebaseVertexAI.SafetyRating.HarmProbability(rawValue: "NEGLIGIBLE"), probabilityScore: 0.048828125, severity: FirebaseVertexAI.SafetyRating.HarmSeverity(rawValue: "HARM_SEVERITY_NEGLIGIBLE"), severityScore: 0.095214844, blocked: false)], finishReason: Optional(FirebaseVertexAI.FinishReason(rawValue: "STOP")), citationMetadata: nil)], promptFeedback: nil, usageMetadata: Optional(FirebaseVertexAI.GenerateContentResponse.UsageMetadata(promptTokenCount: 271, candidatesTokenCount: 7, totalTokenCount: 278)))
} catch {
let errorDescription = String(describing: error)
XCTAssertTrue(errorDescription.contains("403"))
XCTAssertTrue(errorDescription.contains("The caller does not have permission"))
}
}

func testGenerateContent_appCheckNotConfigured_shouldFail() async throws {
let app = try FirebaseApp.defaultNamedCopy(name: TestAppCheckProviderFactory.notConfiguredName)
addTeardownBlock { await app.delete() }
Expand All @@ -83,13 +108,63 @@
let prompt = "Where is Google headquarters located? Answer with the city name only."

do {
_ = try await model.generateContent(prompt)
XCTFail("Expected a Firebase App Check error; none thrown.")
let response = try await model.generateContent(prompt)
XCTFail("Expected a Firebase App Check error, got response: \(response)")
} catch let GenerateContentError.internalError(error) {
XCTAssertTrue(String(describing: error).contains("Firebase App Check token is invalid"))
}
}

// MARK: - Generate Content Streaming

func testGenerateContentStream_text() async throws {
let prompt = "Where is Google headquarters located? Answer with the city name only."

var text = ""
let contentStream = try model.generateContentStream(prompt)
for try await chunk in contentStream {
text += try XCTUnwrap(chunk.text)
}

text = text.trimmingCharacters(in: .whitespacesAndNewlines)
XCTAssertEqual(text, "Mountain View")
}

func testGenerateContentStream_image_fileData_public() async throws {
let storageRef = storage.reference(withPath: "vertexai/public/green.png")
let fileData = FileDataPart(uri: storageRef.gsURI, mimeType: "image/png")

var text = ""
let contentStream = try model.generateContentStream(fileData, "What color is this?")
for try await chunk in contentStream {
text += try XCTUnwrap(chunk.text)
}

text = text.trimmingCharacters(in: .whitespacesAndNewlines)
XCTAssertFalse(text.isEmpty)
}

func testGenerateContentStream_image_fileData_requiresUserAuth_wrongUser_permissionDenied() async throws {
let userID = "3MjEzU6JIobWvHdCYHicnDMcPpQ2"
let storageRef = storage.reference(withPath: "vertexai/authenticated/user/\(userID)/pink.webp")

let fileData = FileDataPart(uri: storageRef.gsURI, mimeType: "image/webp")

let contentStream = try model.generateContentStream(fileData, "What color is this?")

do {
var responses = [GenerateContentResponse]()
for try await response in contentStream {
responses.append(response)
}
XCTFail("Expected to throw an error, got response(s): \(responses)")

Check failure on line 160 in FirebaseVertexAI/Tests/TestApp/Tests/Integration/IntegrationTests.swift

View workflow job for this annotation

GitHub Actions / testapp-integration (iOS, macos-15)

testGenerateContentStream_image_fileData_requiresUserAuth_wrongUser_permissionDenied, failed - Expected to throw an error, got response(s): [FirebaseVertexAI.GenerateContentResponse(candidates: [FirebaseVertexAI.Candidate(content: FirebaseVertexAI.ModelContent(role: Optional("model"), internalParts: [FirebaseVertexAI.ModelContent.InternalPart.text("The")]), safetyRatings: [], finishReason: nil, citationMetadata: nil)], promptFeedback: nil, usageMetadata: Optional(FirebaseVertexAI.GenerateContentResponse.UsageMetadata(promptTokenCount: 271, candidatesTokenCount: 1, totalTokenCount: 272))), FirebaseVertexAI.GenerateContentResponse(candidates: [FirebaseVertexAI.Candidate(content: FirebaseVertexAI.ModelContent(role: Optional("model"), internalParts: [FirebaseVertexAI.ModelContent.InternalPart.text(" color is pink. \n")]), safetyRatings: [FirebaseVertexAI.SafetyRating(category: FirebaseVertexAI.HarmCategory(rawValue: "HARM_CATEGORY_HATE_SPEECH"), probability: FirebaseVertexAI.SafetyRating.HarmProbability(rawValue: "NEGLIGIBLE"), probabilityScore: 0.05493164, severity: FirebaseVertexAI.SafetyRating.HarmSeverity(rawValue: "HARM_SEVERITY_NEGLIGIBLE"), severityScore: 0.1953125, blocked: false), FirebaseVertexAI.SafetyRating(category: FirebaseVertexAI.HarmCategory(rawValue: "HARM_CATEGORY_DANGEROUS_CONTENT"), probability: FirebaseVertexAI.SafetyRating.HarmProbability(rawValue: "NEGLIGIBLE"), probabilityScore: 0.056640625, severity: FirebaseVertexAI.SafetyRating.HarmSeverity(rawValue: "HARM_SEVERITY_NEGLIGIBLE"), severityScore: 0.08154297, blocked: false), FirebaseVertexAI.SafetyRating(category: FirebaseVertexAI.HarmCategory(rawValue: "HARM_CATEGORY_HARASSMENT"), probability: FirebaseVertexAI.SafetyRating.HarmProbability(rawValue: "NEGLIGIBLE"), probabilityScore: 0.057373047, severity: FirebaseVertexAI.SafetyRating.HarmSeverity(rawValue: "HARM_SEVERITY_NEGLIGIBLE"), severityScore: 0.075683594, blocked: false), FirebaseVertexAI.SafetyRating(category: FirebaseVertexAI.HarmCategory(rawValue: "HARM_CATEGORY_SEXUALLY_EXPLICIT"), probability: FirebaseVertexAI.SafetyRating.HarmProbability(rawValue: "NEGLIGIBLE"), probabilityScore: 0.048828125, severity: FirebaseVertexAI.SafetyRating.HarmSeverity(rawValue: "HARM_SEVERITY_NEGLIGIBLE"), severityScore: 0.095214844, blocked: false)], finishReason: Optional(FirebaseVertexAI.FinishReason(rawValue: "STOP")), citationMetadata: nil)], promptFeedback: nil, usageMetadata: Optional(FirebaseVertexAI.GenerateContentResponse.UsageMetadata(promptTokenCount: 271, candidatesTokenCount: 7, totalTokenCount: 278)))]
} catch {
let errorDescription = String(describing: error)
XCTAssertTrue(errorDescription.contains("403"))
XCTAssertTrue(errorDescription.contains("The caller does not have permission"))
}
}

// MARK: - Count Tokens

func testCountTokens_text() async throws {
Expand Down Expand Up @@ -166,8 +241,8 @@
let fileData = FileDataPart(uri: storageRef.gsURI, mimeType: "image/webp")

do {
_ = try await model.countTokens(fileData)
XCTFail("Expected to throw an error.")
let response = try await model.countTokens(fileData)
XCTFail("Expected to throw an error, got response: \(response)")

Check failure on line 245 in FirebaseVertexAI/Tests/TestApp/Tests/Integration/IntegrationTests.swift

View workflow job for this annotation

GitHub Actions / testapp-integration (iOS, macos-15)

testCountTokens_image_fileData_requiresUserAuth_wrongUser_permissionDenied, failed - Expected to throw an error, got response: CountTokensResponse(totalTokens: 266, totalBillableCharacters: Optional(35))
} catch {
let errorDescription = String(describing: error)
XCTAssertTrue(errorDescription.contains("403"))
Expand Down Expand Up @@ -229,8 +304,8 @@
let prompt = "Why is the sky blue?"

do {
_ = try await model.countTokens(prompt)
XCTFail("Expected a Firebase App Check error; none thrown.")
let response = try await model.countTokens(prompt)
XCTFail("Expected a Firebase App Check error, got response: \(response)")
} catch {
XCTAssertTrue(String(describing: error).contains("Firebase App Check token is invalid"))
}
Expand Down
Loading