Skip to content

Commit 1695db3

Browse files
feat(condo): INFRA-91 Open telemetry + Tempo based tracing MVP (#3725)
* feat(condo) INFRA-91: add idcache * feat(condo) INFRA-91: learn how to get send traces to HTTP OTEL COLLECTOR * feat(condo) INFRA-91: learn how to send traces manually * feat(condo) INFRA-91: learn how to send traces via providers * feat(condo) INFRA-91: manually instrument keystone * feat(condo) INFRA-91: add todo delete mut * fix(condo) INFRA-91: fix imports * feat(condo) INFRA-91: move KeystoneInstrumentation.js to other file * feat(condo) INFRA-91: add hooks mechanism * feat(condo) INFRA-91: remove hooks mechanism * feat(condo) INFRA-91: instrument keystone adapter * feat(condo) INFRA-91: add HttpInstrumentation * feat(condo) INFRA-91: add an ability to configure OTEL + fix deps * feat(condo) INFRA-91: add an ability to configure OTEL * fix(condo) INFRA-91: debug why !this.enabled did not work in otel * feat(condo) INFRA-91: remove non-working stuff * refactor(condo) INFRA-91: refactor KeystoneInstrumentation.js * fix(condo) INFRA-91: yarn lock fixes again * fix(condo) INFRA-91: remove ylock * fix(condo) INFRA-91: add new geneerated ylock * fix(condo) INFRA-91: regenerate lockfile * fix(condo) INFRA-91: regenerate lockfile * feat(condo): DOMA-6978 Move holdings out of condo (#3751) * feat(condo): DOMA-6978 exclude holdings from condo organizations * feat(callcenter): DOMA-6978 added create employee CC feature * feat(global): DOMA-6978 Split organization invites * chore(condo): DOMA-6978 Submodule sync (branch) * chore(condo): DOMA-6978 Submodule sync (master) * fix(condo) INFRA-91: message to msg * fix(condo) INFRA-91: fixes on review * fix(condo) INFRA-91: delete KeystoneInstrumentation.js * fix(condo) INFRA-91: move stuff to prepareKeystone.js * fix(condo) INFRA-91: reorder code * fix(condo) INFRA-91: remove Apollo plugin! * fix(condo) INFRA-91: remove Apollo plugin! * fix(condo) INFRA-91: fixes on eslint * fix(condo) INFRA-91: start SDK only when IS_OTEL_TRACING_ENABLED * fix(condo) INFRA-91: add IORedis instrumentation * fix(condo) INFRA-91: remove keystone list tracing * fix(condo) INFRA-91: add @opentelemetry/instrumentation-ioredis --------- Co-authored-by: Matthew <[email protected]>
1 parent 2114cd4 commit 1695db3

File tree

6 files changed

+881
-8
lines changed

6 files changed

+881
-8
lines changed

apps/condo/index.js

+2-2
Original file line numberDiff line numberDiff line change
@@ -130,8 +130,8 @@ const lastApp = conf.NODE_ENV === 'test' ? undefined : new NextApp({ dir: '.' })
130130
const apps = () => {
131131
return [
132132
new HealthCheck({ checks }),
133-
new RequestCache(conf.REQUEST_CACHE_CONFIG ? JSON.parse(conf.REQUEST_CACHE_CONFIG) : {}),
134-
new AdapterCache(conf.ADAPTER_CACHE_CONFIG ? JSON.parse(conf.ADAPTER_CACHE_CONFIG) : {}),
133+
new RequestCache(conf.REQUEST_CACHE_CONFIG ? JSON.parse(conf.REQUEST_CACHE_CONFIG) : { enabled: false }),
134+
new AdapterCache(conf.ADAPTER_CACHE_CONFIG ? JSON.parse(conf.ADAPTER_CACHE_CONFIG) : { enabled: false }),
135135
new VersioningMiddleware(),
136136
new OIDCMiddleware(),
137137
new FeaturesMiddleware(),

apps/condo/package.json

+8
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,14 @@
6666
"@open-condo/next": "workspace:^",
6767
"@open-condo/ui": "workspace:^",
6868
"@open-condo/webhooks": "workspace:^",
69+
"@opentelemetry/api": "^1.4.1",
70+
"@opentelemetry/exporter-metrics-otlp-proto": "^0.41.2",
71+
"@opentelemetry/exporter-trace-otlp-proto": "^0.41.2",
72+
"@opentelemetry/instrumentation-http": "^0.41.2",
73+
"@opentelemetry/instrumentation-ioredis": "^0.35.1",
74+
"@opentelemetry/instrumentation-pg": "^0.36.1",
75+
"@opentelemetry/sdk-metrics": "^1.15.2",
76+
"@opentelemetry/sdk-node": "^0.41.2",
6977
"@tinymce/tinymce-react": "^4.1.0",
7078
"@types/lodash": "4.14.186",
7179
"@types/wavesurfer.js": "^6.0.6",

packages/keystone/KSv5v6/v5/prepareKeystone.js

+2
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ const { registerSchemas } = require('@open-condo/keystone/KSv5v6/v5/registerSche
1414
const { getKeystonePinoOptions, GraphQLLoggerPlugin } = require('@open-condo/keystone/logging')
1515
const { schemaDocPreprocessor, adminDocPreprocessor, escapeSearchPreprocessor, customAccessPostProcessor } = require('@open-condo/keystone/preprocessors')
1616
const { registerTasks } = require('@open-condo/keystone/tasks')
17+
const { KeystoneTracingApp } = require('@open-condo/keystone/tracing')
1718

1819
const { parseCorsSettings } = require('../../cors.utils')
1920
const { expressErrorHandler } = require('../../logging/expressErrorHandler')
@@ -71,6 +72,7 @@ function prepareKeystone ({ onConnect, extendExpressApp, schemas, schemasPreproc
7172
cors: (conf.CORS) ? parseCorsSettings(JSON.parse(conf.CORS)) : { origin: true, credentials: true },
7273
pinoOptions: getKeystonePinoOptions(),
7374
apps: [
75+
new KeystoneTracingApp(),
7476
...((apps) ? apps() : []),
7577
new GraphQLApp({
7678
apollo: {

packages/keystone/adapterCache.js

+1-1
Original file line numberDiff line numberDiff line change
@@ -257,7 +257,7 @@ async function patchKeystoneWithAdapterCache (keystone, cacheAPI) {
257257
const listAdapters = Object.values(keystoneAdapter.listAdapters)
258258

259259
// Step 1: Preprocess lists.
260-
const relations = {} // list -> [{list, path, many}]
260+
const relations = {} // list -> [{list, path, many}]
261261
const manyRefs = new Set() // lists that are referenced in many: true relations
262262
const manyLists = new Set() // lists that have many: true relations
263263

packages/keystone/tracing.js

+150
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,150 @@
1+
const otelApi = require('@opentelemetry/api')
2+
const { OTLPMetricExporter } = require('@opentelemetry/exporter-metrics-otlp-proto')
3+
const { OTLPTraceExporter } = require('@opentelemetry/exporter-trace-otlp-proto')
4+
const { HttpInstrumentation } = require('@opentelemetry/instrumentation-http')
5+
const { IORedisInstrumentation } = require('@opentelemetry/instrumentation-ioredis')
6+
const { PgInstrumentation } = require('@opentelemetry/instrumentation-pg')
7+
const { PeriodicExportingMetricReader } = require('@opentelemetry/sdk-metrics')
8+
const otelSdk = require('@opentelemetry/sdk-node')
9+
const { get } = require('lodash')
10+
11+
const conf = require('@open-condo/config')
12+
13+
const { getLogger } = require('./logging')
14+
15+
const DELIMETER = ':'
16+
const KEYSTONE_MUTATION_QUERY_REGEX = /(?:mutation|query)\s+(\w+)/
17+
18+
const IS_OTEL_TRACING_ENABLED = conf.IS_OTEL_TRACING_ENABLED === '1'
19+
const OTEL_CONFIG = conf.OTEL_CONFIG ? JSON.parse(conf.OTEL_CONFIG) : {}
20+
21+
const { tracesUrl, metricsUrl, headers = {} } = OTEL_CONFIG
22+
23+
const tracers = {}
24+
25+
if (IS_OTEL_TRACING_ENABLED) {
26+
const sdk = new otelSdk.NodeSDK({
27+
serviceName: 'condo',
28+
traceExporter: new OTLPTraceExporter({
29+
url: tracesUrl,
30+
headers: headers,
31+
}),
32+
metricReader: new PeriodicExportingMetricReader({
33+
exporter: new OTLPMetricExporter({
34+
url: metricsUrl,
35+
headers: headers,
36+
concurrencyLimit: 1,
37+
}),
38+
}),
39+
40+
instrumentations: [
41+
new HttpInstrumentation(),
42+
new PgInstrumentation(),
43+
new IORedisInstrumentation,
44+
],
45+
})
46+
47+
sdk.start()
48+
}
49+
50+
const _getTracer = (name) => {
51+
if (!tracers[name]) {
52+
tracers[name] = otelApi.trace.getTracer(
53+
name,
54+
'1.0.0',
55+
)
56+
}
57+
return tracers[name]
58+
}
59+
60+
function _getTracedFunction ({ name, spanHook, tracer, ctx, f }) {
61+
return async function (...args) {
62+
// Sometimes you want the name of the trace to be calculated in runtime
63+
const parsedName = typeof name === 'function' ? name(...args) : name
64+
65+
return tracer.startActiveSpan(parsedName, async (span) => {
66+
spanHook(span, ...args)
67+
68+
const res = await f.call(ctx, ...args)
69+
span.end()
70+
return res
71+
})
72+
}
73+
}
74+
75+
/**
76+
* Monkey patch keystone with open telemetry tracing
77+
*/
78+
class KeystoneTracingApp {
79+
80+
tracer = _getTracer('@open-condo/tracing/keystone-tracing-app')
81+
82+
_getTracedAdapterFunction (tracer, config, ctx, f) {
83+
const { name, listKey } = config
84+
85+
return _getTracedFunction({
86+
name: name + DELIMETER + listKey,
87+
spanHook: (span, _) => {
88+
span.setAttribute('type', 'adapter')
89+
span.setAttribute('listKey', listKey)
90+
span.setAttribute('functionName', name)
91+
},
92+
ctx, f, tracer,
93+
})
94+
}
95+
96+
_patchKeystoneAdapter (tracer, keystone) {
97+
for (const listAdapter of Object.values(get(keystone, ['adapter', 'listAdapters']))) {
98+
const originalListAdapter = listAdapter
99+
const listKey = listAdapter.key
100+
101+
listAdapter.find = this._getTracedAdapterFunction(tracer, { listKey, name: 'adapter:find' }, listAdapter, originalListAdapter.find)
102+
listAdapter.findOne = this._getTracedAdapterFunction(tracer, { listKey, name: 'adapter:findOne' }, listAdapter, originalListAdapter.findOne)
103+
listAdapter.findById = this._getTracedAdapterFunction(tracer, { listKey, name: 'adapter:findById' }, listAdapter, originalListAdapter.findById)
104+
listAdapter.itemsQuery = this._getTracedAdapterFunction(tracer, { listKey, name: 'adapter:itemsQuery' }, listAdapter, originalListAdapter.itemsQuery)
105+
106+
listAdapter.create = this._getTracedAdapterFunction(tracer, { listKey, name: 'adapter:create' }, listAdapter, originalListAdapter.create)
107+
listAdapter.delete = this._getTracedAdapterFunction(tracer, { listKey, name: 'adapter:delete' }, listAdapter, originalListAdapter.delete)
108+
listAdapter.update = this._getTracedAdapterFunction(tracer, { listKey, name: 'adapter:update' }, listAdapter, originalListAdapter.update)
109+
110+
listAdapter.createMany = this._getTracedAdapterFunction(tracer, { listKey, name: 'adapter:createMany' }, listAdapter, originalListAdapter.createMany)
111+
listAdapter.deleteMany = this._getTracedAdapterFunction(tracer, { listKey, name: 'adapter:createMany' }, listAdapter, originalListAdapter.deleteMany)
112+
listAdapter.updateMany = this._getTracedAdapterFunction(tracer, { listKey, name: 'adapter:createMany' }, listAdapter, originalListAdapter.updateMany)
113+
}
114+
}
115+
116+
_patchKeystoneGraphQLExecutor (tracer, keystone) {
117+
const originalExecuteGraphQL = keystone.executeGraphQL
118+
keystone.executeGraphQL = ({ context, query, variables }) => {
119+
let queryName = undefined
120+
if (typeof query === 'string') {
121+
const matches = query.match(KEYSTONE_MUTATION_QUERY_REGEX)
122+
if (matches && matches[1])
123+
queryName = matches[1]
124+
} else {
125+
queryName = get(query, ['definitions', 0, 'name', 'value'])
126+
}
127+
128+
return tracer.startActiveSpan('gql' + DELIMETER + queryName, async (span) => {
129+
span.setAttribute('queryName', queryName)
130+
131+
const result = originalExecuteGraphQL.call(keystone, { context, query, variables })
132+
span.end()
133+
return result
134+
})
135+
}
136+
}
137+
138+
async prepareMiddleware ({ keystone }) {
139+
if (!IS_OTEL_TRACING_ENABLED) return
140+
141+
const tracer = this.tracer
142+
this._patchKeystoneGraphQLExecutor(tracer, keystone)
143+
this._patchKeystoneAdapter(tracer, keystone)
144+
}
145+
}
146+
147+
148+
module.exports = {
149+
KeystoneTracingApp,
150+
}

0 commit comments

Comments
 (0)