From f18224c6869ae52228da3764ca9a427106b872fb Mon Sep 17 00:00:00 2001 From: Sandra G Date: Mon, 5 Aug 2024 13:02:01 -0400 Subject: [PATCH] [Obs AI Assistant] knowledge base integration tests (#189000) Closes https://github.com/elastic/kibana/issues/188999 - integration tests for knowledge base api - adds new config field `modelId`, for internal use, to override elser model id - refactors `knowledgeBaseService.setup()` to fix bug where if the model failed to install when calling ml.putTrainedModel, we dont get stuck polling and retrying the install. We were assuming that the first error that gets throw when the model is exists would only happen once and the return true or false and poll for whether its done installing. But the installation could fail itself causing getTrainedModelsStats to continuously throw and try to install the model. Now user immediately gets error if model fails to install and polling does not happen. --------- Co-authored-by: James Gowdy --- .../server/config.ts | 1 + .../server/plugin.ts | 8 +- .../service/knowledge_base_service/index.ts | 109 +++++--- x-pack/test/functional/services/ml/api.ts | 19 +- .../configs/index.ts | 2 + .../tests/knowledge_base/helpers.ts | 31 +++ .../knowledge_base/knowledge_base.spec.ts | 257 ++++++++++++++++++ .../knowledge_base/knowledge_base.todo.ts | 226 --------------- .../knowledge_base_setup.spec.ts | 36 +++ .../knowledge_base_status.spec.ts | 55 ++++ 10 files changed, 470 insertions(+), 274 deletions(-) create mode 100644 x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/helpers.ts create mode 100644 x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base.spec.ts delete mode 100644 x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base.todo.ts create mode 100644 x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base_setup.spec.ts create mode 100644 x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base_status.spec.ts diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/config.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/config.ts index 61c67ad3e19a9f..abb4590c896967 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/config.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/config.ts @@ -9,6 +9,7 @@ import { schema, type TypeOf } from '@kbn/config-schema'; export const config = schema.object({ enabled: schema.boolean({ defaultValue: true }), + modelId: schema.maybe(schema.string()), }); export type ObservabilityAIAssistantConfig = TypeOf; diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/plugin.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/plugin.ts index 731dd0ce98f331..8a6f414ec92ded 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/plugin.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/plugin.ts @@ -47,10 +47,12 @@ export class ObservabilityAIAssistantPlugin > { logger: Logger; + config: ObservabilityAIAssistantConfig; service: ObservabilityAIAssistantService | undefined; constructor(context: PluginInitializerContext) { this.logger = context.logger.get(); + this.config = context.config.get(); initLangtrace(); } public setup( @@ -112,10 +114,14 @@ export class ObservabilityAIAssistantPlugin // Using once to make sure the same model ID is used during service init and Knowledge base setup const getModelId = once(async () => { + const configModelId = this.config.modelId; + if (configModelId) { + return configModelId; + } const defaultModelId = '.elser_model_2'; const [_, pluginsStart] = await core.getStartServices(); + // Wait for the license to be available so the ML plugin's guards pass once we ask for ELSER stats const license = await firstValueFrom(pluginsStart.licensing.license$); - if (!license.hasAtLeast('enterprise')) { return defaultModelId; } diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/knowledge_base_service/index.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/knowledge_base_service/index.ts index 86a18d5a8efa46..4de6c776661704 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/knowledge_base_service/index.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/knowledge_base_service/index.ts @@ -5,7 +5,7 @@ * 2.0. */ import { errors } from '@elastic/elasticsearch'; -import { serverUnavailable, gatewayTimeout } from '@hapi/boom'; +import { serverUnavailable, gatewayTimeout, badRequest } from '@hapi/boom'; import type { ElasticsearchClient, IUiSettingsClient } from '@kbn/core/server'; import type { Logger } from '@kbn/logging'; import type { TaskManagerStartContract } from '@kbn/task-manager-plugin/server'; @@ -39,14 +39,20 @@ export interface RecalledEntry { labels?: Record; } -function isAlreadyExistsError(error: Error) { +function isModelMissingOrUnavailableError(error: Error) { return ( error instanceof errors.ResponseError && (error.body.error.type === 'resource_not_found_exception' || error.body.error.type === 'status_exception') ); } - +function isCreateModelValidationError(error: Error) { + return ( + error instanceof errors.ResponseError && + error.statusCode === 400 && + error.body?.error?.type === 'action_request_validation_exception' + ); +} function throwKnowledgeBaseNotReady(body: any) { throw serverUnavailable(`Knowledge base is not ready yet`, body); } @@ -84,52 +90,73 @@ export class KnowledgeBaseService { const elserModelId = await this.dependencies.getModelId(); const retryOptions = { factor: 1, minTimeout: 10000, retries: 12 }; - - const installModel = async () => { - this.dependencies.logger.info('Installing ELSER model'); - await this.dependencies.esClient.asInternalUser.ml.putTrainedModel( - { - model_id: elserModelId, - input: { - field_names: ['text_field'], - }, - wait_for_completion: true, - }, - { requestTimeout: '20m' } - ); - this.dependencies.logger.info('Finished installing ELSER model'); - }; - - const getIsModelInstalled = async () => { - const getResponse = await this.dependencies.esClient.asInternalUser.ml.getTrainedModels({ + const getModelInfo = async () => { + return await this.dependencies.esClient.asInternalUser.ml.getTrainedModels({ model_id: elserModelId, include: 'definition_status', }); + }; - this.dependencies.logger.debug( - () => 'Model definition status:\n' + JSON.stringify(getResponse.trained_model_configs[0]) - ); + const isModelInstalledAndReady = async () => { + try { + const getResponse = await getModelInfo(); + this.dependencies.logger.debug( + () => 'Model definition status:\n' + JSON.stringify(getResponse.trained_model_configs[0]) + ); - return Boolean(getResponse.trained_model_configs[0]?.fully_defined); + return Boolean(getResponse.trained_model_configs[0]?.fully_defined); + } catch (error) { + if (isModelMissingOrUnavailableError(error)) { + return false; + } else { + throw error; + } + } }; - await pRetry(async () => { - let isModelInstalled: boolean = false; + const installModelIfDoesNotExist = async () => { + const modelInstalledAndReady = await isModelInstalledAndReady(); + if (!modelInstalledAndReady) { + await installModel(); + } + }; + + const installModel = async () => { + this.dependencies.logger.info('Installing ELSER model'); try { - isModelInstalled = await getIsModelInstalled(); + await this.dependencies.esClient.asInternalUser.ml.putTrainedModel( + { + model_id: elserModelId, + input: { + field_names: ['text_field'], + }, + wait_for_completion: true, + }, + { requestTimeout: '20m' } + ); } catch (error) { - if (isAlreadyExistsError(error)) { - await installModel(); - isModelInstalled = await getIsModelInstalled(); + if (isCreateModelValidationError(error)) { + throw badRequest(error); + } else { + throw error; } } + this.dependencies.logger.info('Finished installing ELSER model'); + }; - if (!isModelInstalled) { - throwKnowledgeBaseNotReady({ - message: 'Model is not fully defined', - }); - } - }, retryOptions); + const pollForModelInstallCompleted = async () => { + await pRetry(async () => { + this.dependencies.logger.info('Polling installation of ELSER model'); + const modelInstalledAndReady = await isModelInstalledAndReady(); + if (!modelInstalledAndReady) { + throwKnowledgeBaseNotReady({ + message: 'Model is not fully defined', + }); + } + }, retryOptions); + }; + await installModelIfDoesNotExist(); + await pollForModelInstallCompleted(); try { await this.dependencies.esClient.asInternalUser.ml.startTrainedModelDeployment({ @@ -139,7 +166,7 @@ export class KnowledgeBaseService { } catch (error) { this.dependencies.logger.debug('Error starting model deployment'); this.dependencies.logger.debug(error); - if (!isAlreadyExistsError(error)) { + if (!isModelMissingOrUnavailableError(error)) { throw error; } } @@ -380,7 +407,7 @@ export class KnowledgeBaseService { namespace, modelId, }).catch((error) => { - if (isAlreadyExistsError(error)) { + if (isModelMissingOrUnavailableError(error)) { throwKnowledgeBaseNotReady(error.body); } throw error; @@ -521,7 +548,7 @@ export class KnowledgeBaseService { })), }; } catch (error) { - if (isAlreadyExistsError(error)) { + if (isModelMissingOrUnavailableError(error)) { throwKnowledgeBaseNotReady(error.body); } throw error; @@ -588,7 +615,7 @@ export class KnowledgeBaseService { return Promise.resolve(); } catch (error) { - if (isAlreadyExistsError(error)) { + if (isModelMissingOrUnavailableError(error)) { throwKnowledgeBaseNotReady(error.body); } throw error; diff --git a/x-pack/test/functional/services/ml/api.ts b/x-pack/test/functional/services/ml/api.ts index a26ac26906468c..8b2c46d474c0c0 100644 --- a/x-pack/test/functional/services/ml/api.ts +++ b/x-pack/test/functional/services/ml/api.ts @@ -1473,11 +1473,13 @@ export function MachineLearningAPIProvider({ getService }: FtrProviderContext) { } }, - async stopTrainedModelDeploymentES(deploymentId: string) { + async stopTrainedModelDeploymentES(deploymentId: string, force: boolean = false) { log.debug(`Stopping trained model deployment with id "${deploymentId}"`); - const { body, status } = await esSupertest.post( - `/_ml/trained_models/${deploymentId}/deployment/_stop` - ); + const url = `/_ml/trained_models/${deploymentId}/deployment/_stop${ + force ? '?force=true' : '' + }`; + + const { body, status } = await esSupertest.post(url); this.assertResponseStatusCode(200, status, body); log.debug('> Trained model deployment stopped'); @@ -1570,8 +1572,13 @@ export function MachineLearningAPIProvider({ getService }: FtrProviderContext) { ); }, - async importTrainedModel(modelId: string, modelName: SupportedTrainedModelNamesType) { - await this.createTrainedModel(modelId, this.getTrainedModelConfig(modelName)); + async importTrainedModel( + modelId: string, + modelName: SupportedTrainedModelNamesType, + config?: PutTrainedModelConfig + ) { + const trainedModelConfig = config ?? this.getTrainedModelConfig(modelName); + await this.createTrainedModel(modelId, trainedModelConfig); await this.createTrainedModelVocabularyES(modelId, this.getTrainedModelVocabulary(modelName)); await this.uploadTrainedModelDefinitionES( modelId, diff --git a/x-pack/test/observability_ai_assistant_api_integration/configs/index.ts b/x-pack/test/observability_ai_assistant_api_integration/configs/index.ts index 75f7bb628b4bef..74f0016f009f76 100644 --- a/x-pack/test/observability_ai_assistant_api_integration/configs/index.ts +++ b/x-pack/test/observability_ai_assistant_api_integration/configs/index.ts @@ -8,6 +8,7 @@ import { mapValues } from 'lodash'; import path from 'path'; import { createTestConfig, CreateTestConfig } from '../common/config'; +import { SUPPORTED_TRAINED_MODELS } from '../../functional/services/ml/api'; export const observabilityAIAssistantDebugLogger = { name: 'plugins.observabilityAIAssistant', @@ -30,6 +31,7 @@ export const observabilityAIAssistantFtrConfigs = { __dirname, '../../../../test/analytics/plugins/analytics_ftr_helpers' ), + 'xpack.observabilityAIAssistant.modelId': SUPPORTED_TRAINED_MODELS.TINY_ELSER.name, }, }, }; diff --git a/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/helpers.ts b/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/helpers.ts new file mode 100644 index 00000000000000..1d9a9170e56eac --- /dev/null +++ b/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/helpers.ts @@ -0,0 +1,31 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { MachineLearningProvider } from '../../../api_integration/services/ml'; +import { SUPPORTED_TRAINED_MODELS } from '../../../functional/services/ml/api'; + +export const TINY_ELSER = { + ...SUPPORTED_TRAINED_MODELS.TINY_ELSER, + id: SUPPORTED_TRAINED_MODELS.TINY_ELSER.name, +}; + +export async function createKnowledgeBaseModel(ml: ReturnType) { + const config = { + ...ml.api.getTrainedModelConfig(TINY_ELSER.name), + input: { + field_names: ['text_field'], + }, + }; + await ml.api.importTrainedModel(TINY_ELSER.name, TINY_ELSER.id, config); + await ml.api.assureMlStatsIndexExists(); +} +export async function deleteKnowledgeBaseModel(ml: ReturnType) { + await ml.api.stopTrainedModelDeploymentES(TINY_ELSER.id, true); + await ml.api.deleteTrainedModelES(TINY_ELSER.id); + await ml.api.cleanMlIndices(); + await ml.testResources.cleanMLSavedObjects(); +} diff --git a/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base.spec.ts b/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base.spec.ts new file mode 100644 index 00000000000000..74ba11ce2eb9ea --- /dev/null +++ b/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base.spec.ts @@ -0,0 +1,257 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; +import { FtrProviderContext } from '../../common/ftr_provider_context'; +import { createKnowledgeBaseModel, deleteKnowledgeBaseModel } from './helpers'; + +interface KnowledgeBaseEntry { + id: string; + text: string; +} + +export default function ApiTest({ getService }: FtrProviderContext) { + const ml = getService('ml'); + const es = getService('es'); + + const observabilityAIAssistantAPIClient = getService('observabilityAIAssistantAPIClient'); + const KB_INDEX = '.kibana-observability-ai-assistant-kb-*'; + + describe('Knowledge base', () => { + before(async () => { + await createKnowledgeBaseModel(ml); + }); + + after(async () => { + await deleteKnowledgeBaseModel(ml); + }); + + it('returns 200 on knowledge base setup', async () => { + const res = await observabilityAIAssistantAPIClient + .editorUser({ + endpoint: 'POST /internal/observability_ai_assistant/kb/setup', + }) + .expect(200); + expect(res.body).to.eql({}); + }); + describe('when managing a single entry', () => { + const knowledgeBaseEntry = { + id: 'my-doc-id-1', + text: 'My content', + }; + it('returns 200 on create', async () => { + await observabilityAIAssistantAPIClient + .editorUser({ + endpoint: 'POST /internal/observability_ai_assistant/kb/entries/save', + params: { body: knowledgeBaseEntry }, + }) + .expect(200); + const res = await observabilityAIAssistantAPIClient.editorUser({ + endpoint: 'GET /internal/observability_ai_assistant/kb/entries', + params: { + query: { + query: '', + sortBy: 'doc_id', + sortDirection: 'asc', + }, + }, + }); + const entry = res.body.entries[0]; + expect(entry.id).to.equal(knowledgeBaseEntry.id); + expect(entry.text).to.equal(knowledgeBaseEntry.text); + }); + + it('returns 200 on get entries and entry exists', async () => { + const res = await observabilityAIAssistantAPIClient + .editorUser({ + endpoint: 'GET /internal/observability_ai_assistant/kb/entries', + params: { + query: { + query: '', + sortBy: 'doc_id', + sortDirection: 'asc', + }, + }, + }) + .expect(200); + const entry = res.body.entries[0]; + expect(entry.id).to.equal(knowledgeBaseEntry.id); + expect(entry.text).to.equal(knowledgeBaseEntry.text); + }); + + it('returns 200 on delete', async () => { + const entryId = 'my-doc-id-1'; + await observabilityAIAssistantAPIClient + .editorUser({ + endpoint: 'DELETE /internal/observability_ai_assistant/kb/entries/{entryId}', + params: { + path: { entryId }, + }, + }) + .expect(200); + + const res = await observabilityAIAssistantAPIClient + .editorUser({ + endpoint: 'GET /internal/observability_ai_assistant/kb/entries', + params: { + query: { + query: '', + sortBy: 'doc_id', + sortDirection: 'asc', + }, + }, + }) + .expect(200); + expect( + res.body.entries.filter((entry: KnowledgeBaseEntry) => entry.id.startsWith('my-doc-id')) + .length + ).to.eql(0); + }); + + it('returns 500 on delete not found', async () => { + const entryId = 'my-doc-id-1'; + await observabilityAIAssistantAPIClient + .editorUser({ + endpoint: 'DELETE /internal/observability_ai_assistant/kb/entries/{entryId}', + params: { + path: { entryId }, + }, + }) + .expect(500); + }); + }); + describe('when managing multiple entries', () => { + before(async () => { + es.deleteByQuery({ + index: KB_INDEX, + conflicts: 'proceed', + query: { match_all: {} }, + }); + }); + afterEach(async () => { + es.deleteByQuery({ + index: KB_INDEX, + conflicts: 'proceed', + query: { match_all: {} }, + }); + }); + const knowledgeBaseEntries: KnowledgeBaseEntry[] = [ + { + id: 'my_doc_a', + text: 'My content a', + }, + { + id: 'my_doc_b', + text: 'My content b', + }, + { + id: 'my_doc_c', + text: 'My content c', + }, + ]; + it('returns 200 on create', async () => { + await observabilityAIAssistantAPIClient + .editorUser({ + endpoint: 'POST /internal/observability_ai_assistant/kb/entries/import', + params: { body: { entries: knowledgeBaseEntries } }, + }) + .expect(200); + + const res = await observabilityAIAssistantAPIClient + .editorUser({ + endpoint: 'GET /internal/observability_ai_assistant/kb/entries', + params: { + query: { + query: '', + sortBy: 'doc_id', + sortDirection: 'asc', + }, + }, + }) + .expect(200); + expect( + res.body.entries.filter((entry: KnowledgeBaseEntry) => entry.id.startsWith('my_doc')) + .length + ).to.eql(3); + }); + + it('allows sorting', async () => { + await observabilityAIAssistantAPIClient + .editorUser({ + endpoint: 'POST /internal/observability_ai_assistant/kb/entries/import', + params: { body: { entries: knowledgeBaseEntries } }, + }) + .expect(200); + + const res = await observabilityAIAssistantAPIClient + .editorUser({ + endpoint: 'GET /internal/observability_ai_assistant/kb/entries', + params: { + query: { + query: '', + sortBy: 'doc_id', + sortDirection: 'desc', + }, + }, + }) + .expect(200); + + const entries = res.body.entries.filter((entry: KnowledgeBaseEntry) => + entry.id.startsWith('my_doc') + ); + expect(entries[0].id).to.eql('my_doc_c'); + expect(entries[1].id).to.eql('my_doc_b'); + expect(entries[2].id).to.eql('my_doc_a'); + + // asc + const resAsc = await observabilityAIAssistantAPIClient + .editorUser({ + endpoint: 'GET /internal/observability_ai_assistant/kb/entries', + params: { + query: { + query: '', + sortBy: 'doc_id', + sortDirection: 'asc', + }, + }, + }) + .expect(200); + + const entriesAsc = resAsc.body.entries.filter((entry: KnowledgeBaseEntry) => + entry.id.startsWith('my_doc') + ); + expect(entriesAsc[0].id).to.eql('my_doc_a'); + expect(entriesAsc[1].id).to.eql('my_doc_b'); + expect(entriesAsc[2].id).to.eql('my_doc_c'); + }); + it('allows searching', async () => { + await observabilityAIAssistantAPIClient + .editorUser({ + endpoint: 'POST /internal/observability_ai_assistant/kb/entries/import', + params: { body: { entries: knowledgeBaseEntries } }, + }) + .expect(200); + + const res = await observabilityAIAssistantAPIClient + .editorUser({ + endpoint: 'GET /internal/observability_ai_assistant/kb/entries', + params: { + query: { + query: 'my_doc_a', + sortBy: 'doc_id', + sortDirection: 'asc', + }, + }, + }) + .expect(200); + + expect(res.body.entries.length).to.eql(1); + expect(res.body.entries[0].id).to.eql('my_doc_a'); + }); + }); + }); +} diff --git a/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base.todo.ts b/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base.todo.ts deleted file mode 100644 index dc1c34fbdd4c29..00000000000000 --- a/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base.todo.ts +++ /dev/null @@ -1,226 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import expect from '@kbn/expect'; -import getPort from 'get-port'; -import http, { Server } from 'http'; -import { FtrProviderContext } from '../../common/ftr_provider_context'; - -/* - This test is disabled because the Knowledge base requires a trained model (ELSER) - which is not available in FTR tests. - - When a comparable, less expensive trained model is available, this test should be re-enabled. -*/ - -export default function ApiTest({ getService }: FtrProviderContext) { - const supertest = getService('supertest'); - - const KNOWLEDGE_BASE_API_URL = `/internal/observability_ai_assistant/kb`; - - describe('Knowledge base', () => { - const requestHandler = ( - request: http.IncomingMessage, - response: http.ServerResponse & { req: http.IncomingMessage } - ) => {}; - - let server: Server; - - before(async () => { - const port = await getPort({ port: getPort.makeRange(9000, 9100) }); - - server = http - .createServer((request, response) => { - requestHandler(request, response); - }) - .listen(port); - }); - - after(() => { - server.close(); - }); - - it('should be possible to set up the knowledge base', async () => { - return supertest - .get(`${KNOWLEDGE_BASE_API_URL}/setup`) - .set('kbn-xsrf', 'foo') - .expect(200) - .then((response) => { - expect(response.body).to.eql({ entries: [] }); - }); - }); - - describe('when creating a single entry', () => { - it('returns a 200 when using the right payload', async () => { - const knowledgeBaseEntry = { - id: 'my-doc-id-1', - text: 'My content', - }; - - await supertest - .post(`${KNOWLEDGE_BASE_API_URL}/entries/save`) - .set('kbn-xsrf', 'foo') - .send(knowledgeBaseEntry) - .expect(200); - - return supertest - .get(`${KNOWLEDGE_BASE_API_URL}/entries?query=&sortBy=doc_id&sortDirection=asc`) - .set('kbn-xsrf', 'foo') - .expect(200) - .then((response) => { - expect(response.body).to.eql({ entries: [knowledgeBaseEntry] }); - }); - }); - - it('returns a 500 when using the wrong payload', async () => { - const knowledgeBaseEntry = { - foo: 'my-doc-id-1', - }; - - await supertest - .post(`${KNOWLEDGE_BASE_API_URL}/entries/save`) - .set('kbn-xsrf', 'foo') - .send(knowledgeBaseEntry) - .expect(500); - }); - }); - - describe('when importing multiple entries', () => { - it('returns a 200 when using the right payload', async () => { - const knowledgeBaseEntries = [ - { - id: 'my-doc-id-2', - text: 'My content 2', - }, - { - id: 'my-doc-id-3', - text: 'My content 3', - }, - ]; - - await supertest - .post(`${KNOWLEDGE_BASE_API_URL}/entries/import`) - .set('kbn-xsrf', 'foo') - .send(knowledgeBaseEntries) - .expect(200); - - return supertest - .get(`${KNOWLEDGE_BASE_API_URL}/entries?query=&sortBy=doc_id&sortDirection=asc`) - .set('kbn-xsrf', 'foo') - .expect(200) - .then((response) => { - expect(response.body).to.eql({ entries: knowledgeBaseEntries }); - }); - }); - - it('returns a 500 when using the wrong payload', async () => { - const knowledgeBaseEntry = { - foo: 'my-doc-id-1', - }; - - await supertest - .post(`${KNOWLEDGE_BASE_API_URL}/entries/import`) - .set('kbn-xsrf', 'foo') - .send(knowledgeBaseEntry) - .expect(500); - }); - }); - - describe('when deleting an entry', () => { - it('returns a 200 when the item is found and the item is deleted', async () => { - await supertest - .delete(`${KNOWLEDGE_BASE_API_URL}/entries/delete/my-doc-id-2`) - .set('kbn-xsrf', 'foo') - .expect(200); - - return supertest - .get(`${KNOWLEDGE_BASE_API_URL}/entries?query=&sortBy=doc_id&sortDirection=asc`) - .set('kbn-xsrf', 'foo') - .expect(200) - .then((response) => { - expect(response.body).to.eql({ - entries: [ - { - id: 'my-doc-id-1', - text: 'My content 1', - confidence: 'high', - }, - { - id: 'my-doc-id-3', - text: 'My content 3', - }, - ], - }); - }); - }); - - it('returns a 500 when the item is not found ', async () => { - return await supertest - .delete(`${KNOWLEDGE_BASE_API_URL}/entries/delete/my-doc-id-2`) - .set('kbn-xsrf', 'foo') - .expect(500); - }); - }); - - describe('when retrieving entries', () => { - it('returns a 200 when calling get entries with the right parameters', async () => { - return supertest - .get(`${KNOWLEDGE_BASE_API_URL}/entries?query=&sortBy=doc_id&sortDirection=asc`) - .set('kbn-xsrf', 'foo') - .expect(200) - .then((response) => { - expect(response.body).to.eql({ entries: [] }); - }); - }); - - it('allows sorting', async () => { - return supertest - .get(`${KNOWLEDGE_BASE_API_URL}/entries?query=&sortBy=doc_id&sortDirection=desc`) - .set('kbn-xsrf', 'foo') - .expect(200) - .then((response) => { - expect(response.body).to.eql({ - entries: [ - { - id: 'my-doc-id-3', - text: 'My content 3', - }, - { - id: 'my-doc-id-1', - text: 'My content 1', - }, - ], - }); - }); - }); - - it('allows searching', async () => { - return supertest - .get(`${KNOWLEDGE_BASE_API_URL}/entries?query=my-doc-3&sortBy=doc_id&sortDirection=asc`) - .set('kbn-xsrf', 'foo') - .expect(200) - .then((response) => { - expect(response.body).to.eql({ - entries: [ - { - id: 'my-doc-id-3', - text: 'My content 3', - }, - ], - }); - }); - }); - - it('returns a 500 when calling get entries with the wrong parameters', async () => { - return supertest - .get(`${KNOWLEDGE_BASE_API_URL}/entries`) - .set('kbn-xsrf', 'foo') - .expect(500); - }); - }); - }); -} diff --git a/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base_setup.spec.ts b/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base_setup.spec.ts new file mode 100644 index 00000000000000..9099eff540d35c --- /dev/null +++ b/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base_setup.spec.ts @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; +import { FtrProviderContext } from '../../common/ftr_provider_context'; +import { deleteKnowledgeBaseModel, createKnowledgeBaseModel } from './helpers'; + +export default function ApiTest({ getService }: FtrProviderContext) { + const ml = getService('ml'); + const observabilityAIAssistantAPIClient = getService('observabilityAIAssistantAPIClient'); + + describe('/internal/observability_ai_assistant/kb/setup', () => { + it('returns empty object when successful', async () => { + await createKnowledgeBaseModel(ml); + const res = await observabilityAIAssistantAPIClient + .editorUser({ + endpoint: 'POST /internal/observability_ai_assistant/kb/setup', + }) + .expect(200); + expect(res.body).to.eql({}); + await deleteKnowledgeBaseModel(ml); + }); + + it('returns bad request if model cannot be installed', async () => { + await observabilityAIAssistantAPIClient + .editorUser({ + endpoint: 'POST /internal/observability_ai_assistant/kb/setup', + }) + .expect(400); + }); + }); +} diff --git a/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base_status.spec.ts b/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base_status.spec.ts new file mode 100644 index 00000000000000..62d2ab7cae785a --- /dev/null +++ b/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base_status.spec.ts @@ -0,0 +1,55 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; +import { FtrProviderContext } from '../../common/ftr_provider_context'; +import { deleteKnowledgeBaseModel, createKnowledgeBaseModel, TINY_ELSER } from './helpers'; + +export default function ApiTest({ getService }: FtrProviderContext) { + const ml = getService('ml'); + const observabilityAIAssistantAPIClient = getService('observabilityAIAssistantAPIClient'); + + describe('/internal/observability_ai_assistant/kb/status', () => { + before(async () => { + await createKnowledgeBaseModel(ml); + await observabilityAIAssistantAPIClient + .editorUser({ + endpoint: 'POST /internal/observability_ai_assistant/kb/setup', + }) + .expect(200); + }); + + after(async () => { + await deleteKnowledgeBaseModel(ml); + }); + + it('returns correct status after knowledge base is setup', async () => { + const res = await observabilityAIAssistantAPIClient + .editorUser({ + endpoint: 'GET /internal/observability_ai_assistant/kb/status', + }) + .expect(200); + expect(res.body.deployment_state).to.eql('started'); + expect(res.body.model_name).to.eql(TINY_ELSER.id); + }); + + it('returns correct status after elser is stopped', async () => { + await ml.api.stopTrainedModelDeploymentES(TINY_ELSER.id, true); + + const res = await observabilityAIAssistantAPIClient + .editorUser({ + endpoint: 'GET /internal/observability_ai_assistant/kb/status', + }) + .expect(200); + + expect(res.body).to.eql({ + ready: false, + model_name: TINY_ELSER.id, + }); + }); + }); +}