Skip to content

Commit d4a257b

Browse files
ivanahuckovaivanahuckova
authored
Logs: Update logic to process logs dataPlane frame with labels field (#77708)
* Logs: Update dataplane logic to use labels instead of attributes * Update Loki logs data plane data frame according to specs * Remove only in test
1 parent f9fffd3 commit d4a257b

15 files changed

+57
-57
lines changed

‎pkg/tsdb/loki/frame.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -173,7 +173,7 @@ func adjustDataplaneLogsFrame(frame *data.Frame, query *lokiQuery) error {
173173
}
174174

175175
timeField.Name = "timestamp"
176-
labelsField.Name = "attributes"
176+
labelsField.Name = "labels"
177177
lineField.Name = "body"
178178

179179
if frame.Meta == nil {

‎pkg/tsdb/loki/testdata_dataplane/streams_parse_errors.golden.jsonc

+2-2
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@
1111
// Name:
1212
// Dimensions: 4 Fields by 4 Rows
1313
// +------------------------------------------------+-------------------------------+----------------+------------------------------+
14-
// | Name: attributes | Name: timestamp | Name: body | Name: id |
14+
// | Name: labels | Name: timestamp | Name: body | Name: id |
1515
// | Labels: | Labels: | Labels: | Labels: |
1616
// | Type: []json.RawMessage | Type: []time.Time | Type: []string | Type: []string |
1717
// +------------------------------------------------+-------------------------------+----------------+------------------------------+
@@ -38,7 +38,7 @@
3838
},
3939
"fields": [
4040
{
41-
"name": "attributes",
41+
"name": "labels",
4242
"type": "other",
4343
"typeInfo": {
4444
"frame": "json.RawMessage"

‎pkg/tsdb/loki/testdata_dataplane/streams_simple.golden.jsonc

+2-2
Original file line numberDiff line numberDiff line change
@@ -119,7 +119,7 @@
119119
// Name:
120120
// Dimensions: 4 Fields by 6 Rows
121121
// +---------------------------------------+-----------------------------------------+------------------+--------------------------------+
122-
// | Name: attributes | Name: timestamp | Name: body | Name: id |
122+
// | Name: labels | Name: timestamp | Name: body | Name: id |
123123
// | Labels: | Labels: | Labels: | Labels: |
124124
// | Type: []json.RawMessage | Type: []time.Time | Type: []string | Type: []string |
125125
// +---------------------------------------+-----------------------------------------+------------------+--------------------------------+
@@ -256,7 +256,7 @@
256256
},
257257
"fields": [
258258
{
259-
"name": "attributes",
259+
"name": "labels",
260260
"type": "other",
261261
"typeInfo": {
262262
"frame": "json.RawMessage"

‎pkg/tsdb/loki/testdata_logs_dataplane/streams_parse_errors.golden.jsonc

+2-2
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@
1111
// Name:
1212
// Dimensions: 4 Fields by 4 Rows
1313
// +------------------------------------------------+-------------------------------+----------------+------------------------------+
14-
// | Name: attributes | Name: timestamp | Name: body | Name: id |
14+
// | Name: labels | Name: timestamp | Name: body | Name: id |
1515
// | Labels: | Labels: | Labels: | Labels: |
1616
// | Type: []json.RawMessage | Type: []time.Time | Type: []string | Type: []string |
1717
// +------------------------------------------------+-------------------------------+----------------+------------------------------+
@@ -38,7 +38,7 @@
3838
},
3939
"fields": [
4040
{
41-
"name": "attributes",
41+
"name": "labels",
4242
"type": "other",
4343
"typeInfo": {
4444
"frame": "json.RawMessage"

‎pkg/tsdb/loki/testdata_logs_dataplane/streams_simple.golden.jsonc

+2-2
Original file line numberDiff line numberDiff line change
@@ -119,7 +119,7 @@
119119
// Name:
120120
// Dimensions: 4 Fields by 6 Rows
121121
// +---------------------------------------+-----------------------------------------+------------------+--------------------------------+
122-
// | Name: attributes | Name: timestamp | Name: body | Name: id |
122+
// | Name: labels | Name: timestamp | Name: body | Name: id |
123123
// | Labels: | Labels: | Labels: | Labels: |
124124
// | Type: []json.RawMessage | Type: []time.Time | Type: []string | Type: []string |
125125
// +---------------------------------------+-----------------------------------------+------------------+--------------------------------+
@@ -256,7 +256,7 @@
256256
},
257257
"fields": [
258258
{
259-
"name": "attributes",
259+
"name": "labels",
260260
"type": "other",
261261
"typeInfo": {
262262
"frame": "json.RawMessage"

‎public/app/features/explore/Logs/LogsTable.test.tsx

+2-2
Original file line numberDiff line numberDiff line change
@@ -205,11 +205,11 @@ describe('LogsTable', () => {
205205
});
206206
});
207207

208-
it('should not render `attributes`', async () => {
208+
it('should not render `labels`', async () => {
209209
setup(undefined, getMockLokiFrameDataPlane());
210210

211211
await waitFor(() => {
212-
const columns = screen.queryAllByRole('columnheader', { name: 'attributes' });
212+
const columns = screen.queryAllByRole('columnheader', { name: 'labels' });
213213

214214
expect(columns.length).toBe(0);
215215
});

‎public/app/features/explore/Logs/LogsTable.tsx

+6-6
Original file line numberDiff line numberDiff line change
@@ -181,12 +181,12 @@ const isFieldFilterable = (field: Field, logsFrame?: LogsFrame | undefined) => {
181181
function extractFieldsAndExclude(dataFrame: DataFrame) {
182182
return dataFrame.fields
183183
.filter((field: Field & { typeInfo?: { frame: string } }) => {
184-
const isFieldLokiLabels = field.typeInfo?.frame === 'json.RawMessage' && field.name === 'labels';
184+
const isFieldLokiLabels =
185+
field.typeInfo?.frame === 'json.RawMessage' &&
186+
field.name === 'labels' &&
187+
dataFrame?.meta?.type !== DataFrameType.LogLines;
185188
const isFieldDataplaneLabels =
186-
field.name === 'attributes' &&
187-
field.type === FieldType.other &&
188-
dataFrame?.meta?.type === DataFrameType.LogLines;
189-
189+
field.name === 'labels' && field.type === FieldType.other && dataFrame?.meta?.type === DataFrameType.LogLines;
190190
return isFieldLokiLabels || isFieldDataplaneLabels;
191191
})
192192
.flatMap((field: Field) => {
@@ -241,7 +241,7 @@ function buildLabelFilters(columnsWithMeta: Record<string, fieldNameMeta>, logsF
241241

242242
// We could be getting fresh data
243243
const uniqueLabels = new Set<string>();
244-
const logFrameLabels = logsFrame?.getAttributesAsLabels();
244+
const logFrameLabels = logsFrame?.getLogFrameLabelsAsLabels();
245245

246246
// Populate the set with all labels from latest dataframe
247247
logFrameLabels?.forEach((labels) => {

‎public/app/features/explore/Logs/LogsTableWrap.tsx

+1-1
Original file line numberDiff line numberDiff line change
@@ -98,7 +98,7 @@ export function LogsTableWrap(props: Props) {
9898
useEffect(() => {
9999
const numberOfLogLines = dataFrame ? dataFrame.length : 0;
100100
const logsFrame = parseLogsFrame(dataFrame);
101-
const labels = logsFrame?.getAttributesAsLabels();
101+
const labels = logsFrame?.getLogFrameLabelsAsLabels();
102102

103103
const otherFields = logsFrame ? logsFrame.extraFields.filter((field) => !field?.config?.custom?.hidden) : [];
104104
if (logsFrame?.severityField) {

‎public/app/features/explore/Logs/utils/testMocks.test.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,7 @@ export const getMockLokiFrameDataPlane = (override?: Partial<DataFrame>): DataFr
6060
fields: [
6161
{
6262
config: {},
63-
name: 'attributes',
63+
name: 'labels',
6464
type: FieldType.other,
6565
values: [
6666
{ app: 'grafana', cluster: 'dev-us-central-0', container: 'hg-plugins' },

‎public/app/features/logs/components/logParser.test.ts

+3-3
Original file line numberDiff line numberDiff line change
@@ -267,7 +267,7 @@ describe('logParser', () => {
267267
},
268268
{
269269
config: {},
270-
name: 'attributes',
270+
name: 'labels',
271271
type: FieldType.other,
272272
values: [{ a: 1, b: 2 }],
273273
},
@@ -315,7 +315,7 @@ describe('logParser', () => {
315315
},
316316
{
317317
config: { links },
318-
name: 'attributes',
318+
name: 'labels',
319319
type: FieldType.other,
320320
values: [{ a: 1, b: 2 }],
321321
},
@@ -335,7 +335,7 @@ describe('logParser', () => {
335335
expectHasField(output, 'timestamp');
336336
expectHasField(output, 'body');
337337
expectHasField(output, 'id');
338-
expectHasField(output, 'attributes');
338+
expectHasField(output, 'labels');
339339
expectHasField(output, 'severity');
340340
});
341341

‎public/app/features/logs/legacyLogsFrame.ts

+2-2
Original file line numberDiff line numberDiff line change
@@ -68,8 +68,8 @@ export function parseLegacyLogsFrame(frame: DataFrame): LogsFrame | null {
6868
timeNanosecondField,
6969
severityField,
7070
idField,
71-
getAttributes: getL,
72-
getAttributesAsLabels: getL,
71+
getLogFrameLabels: getL,
72+
getLogFrameLabelsAsLabels: getL,
7373
extraFields,
7474
};
7575
}

‎public/app/features/logs/logsFrame.test.ts

+18-18
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
import { FieldType, DataFrameType, Field, Labels } from '@grafana/data';
22

3-
import { parseLogsFrame, attributesToLabels } from './logsFrame';
3+
import { parseLogsFrame, logFrameLabelsToLabels } from './logsFrame';
44

55
function makeString(name: string, values: string[], labels?: Labels): Field {
66
return {
@@ -36,7 +36,7 @@ describe('parseLogsFrame should parse different logs-dataframe formats', () => {
3636
const body = makeString('body', ['line1', 'line2']);
3737
const severity = makeString('severity', ['info', 'debug']);
3838
const id = makeString('id', ['id1', 'id2']);
39-
const attributes = makeObject('attributes', [
39+
const labels = makeObject('labels', [
4040
{ counter: '38141', label: 'val2', level: 'warning', nested: { a: '1', b: ['2', '3'] } },
4141
{ counter: '38143', label: 'val2', level: 'info', nested: { a: '11', b: ['12', '13'] } },
4242
]);
@@ -45,7 +45,7 @@ describe('parseLogsFrame should parse different logs-dataframe formats', () => {
4545
meta: {
4646
type: DataFrameType.LogLines,
4747
},
48-
fields: [id, body, attributes, severity, time],
48+
fields: [id, body, labels, severity, time],
4949
length: 2,
5050
});
5151

@@ -56,11 +56,11 @@ describe('parseLogsFrame should parse different logs-dataframe formats', () => {
5656
expect(result!.idField?.values[0]).toBe(id.values[0]);
5757
expect(result!.timeNanosecondField).toBeNull();
5858
expect(result!.severityField?.values[0]).toBe(severity.values[0]);
59-
expect(result!.getAttributes()).toStrictEqual([
59+
expect(result!.getLogFrameLabels()).toStrictEqual([
6060
{ counter: '38141', label: 'val2', level: 'warning', nested: { a: '1', b: ['2', '3'] } },
6161
{ counter: '38143', label: 'val2', level: 'info', nested: { a: '11', b: ['12', '13'] } },
6262
]);
63-
expect(result!.getAttributesAsLabels()).toStrictEqual([
63+
expect(result!.getLogFrameLabelsAsLabels()).toStrictEqual([
6464
{ counter: '38141', label: 'val2', level: 'warning', nested: `{"a":"1","b":["2","3"]}` },
6565
{ counter: '38143', label: 'val2', level: 'info', nested: `{"a":"11","b":["12","13"]}` },
6666
]);
@@ -85,11 +85,11 @@ describe('parseLogsFrame should parse different logs-dataframe formats', () => {
8585
expect(result!.idField?.values[0]).toBe(id.values[0]);
8686
expect(result!.timeNanosecondField?.values[0]).toBe(ns.values[0]);
8787
expect(result!.severityField).toBeNull();
88-
expect(result!.getAttributes()).toStrictEqual([
88+
expect(result!.getLogFrameLabels()).toStrictEqual([
8989
{ counter: '34543', lable: 'val3', level: 'info' },
9090
{ counter: '34543', lable: 'val3', level: 'info' },
9191
]);
92-
expect(result!.getAttributesAsLabels()).toStrictEqual([
92+
expect(result!.getLogFrameLabelsAsLabels()).toStrictEqual([
9393
{ counter: '34543', lable: 'val3', level: 'info' },
9494
{ counter: '34543', lable: 'val3', level: 'info' },
9595
]);
@@ -123,11 +123,11 @@ describe('parseLogsFrame should parse different logs-dataframe formats', () => {
123123
expect(result!.idField?.values[0]).toBe(id.values[0]);
124124
expect(result!.timeNanosecondField?.values[0]).toBe(ns.values[0]);
125125
expect(result!.severityField).toBeNull();
126-
expect(result!.getAttributes()).toStrictEqual([
126+
expect(result!.getLogFrameLabels()).toStrictEqual([
127127
{ counter: '38141', label: 'val2', level: 'warning' },
128128
{ counter: '38143', label: 'val2', level: 'info' },
129129
]);
130-
expect(result!.getAttributesAsLabels()).toStrictEqual([
130+
expect(result!.getLogFrameLabelsAsLabels()).toStrictEqual([
131131
{ counter: '38141', label: 'val2', level: 'warning' },
132132
{ counter: '38143', label: 'val2', level: 'info' },
133133
]);
@@ -161,8 +161,8 @@ describe('parseLogsFrame should parse different logs-dataframe formats', () => {
161161
expect(result!.severityField?.values[0]).toBe(level.values[0]);
162162
expect(result!.idField).toBeNull();
163163
expect(result!.timeNanosecondField).toBeNull();
164-
expect(result!.getAttributesAsLabels()).toBeNull();
165-
expect(result!.getAttributes()).toBeNull();
164+
expect(result!.getLogFrameLabelsAsLabels()).toBeNull();
165+
expect(result!.getLogFrameLabels()).toBeNull();
166166
expect(result?.extraFields.map((f) => f.name)).toStrictEqual(['_source', 'hostname']);
167167
});
168168

@@ -182,8 +182,8 @@ describe('parseLogsFrame should parse different logs-dataframe formats', () => {
182182
expect(result!.severityField).toBeNull();
183183
expect(result!.idField).toBeNull();
184184
expect(result!.timeNanosecondField).toBeNull();
185-
expect(result!.getAttributesAsLabels()).toBeNull();
186-
expect(result!.getAttributes()).toBeNull();
185+
expect(result!.getLogFrameLabelsAsLabels()).toBeNull();
186+
expect(result!.getLogFrameLabels()).toBeNull();
187187
expect(result?.extraFields).toStrictEqual([]);
188188
});
189189

@@ -208,15 +208,15 @@ describe('parseLogsFrame should parse different logs-dataframe formats', () => {
208208
expect(result!.severityField).toBeNull();
209209
expect(result!.idField).toBeNull();
210210
expect(result!.timeNanosecondField).toBeNull();
211-
expect(result!.getAttributesAsLabels()).toBeNull();
212-
expect(result!.getAttributes()).toBeNull();
211+
expect(result!.getLogFrameLabelsAsLabels()).toBeNull();
212+
expect(result!.getLogFrameLabels()).toBeNull();
213213
});
214214
});
215215

216-
describe('attributesToLabels', () => {
216+
describe('logFrameLabelsToLabels', () => {
217217
it('should convert nested structures correctly', () => {
218218
expect(
219-
attributesToLabels({
219+
logFrameLabelsToLabels({
220220
key1: 'val1',
221221
key2: ['k2v1', 'k2v2', 'k2v3'],
222222
key3: {
@@ -240,7 +240,7 @@ describe('attributesToLabels', () => {
240240

241241
it('should convert not-nested structures correctly', () => {
242242
expect(
243-
attributesToLabels({
243+
logFrameLabelsToLabels({
244244
key1: 'val1',
245245
key2: 'val2',
246246
})

‎public/app/features/logs/logsFrame.ts

+12-12
Original file line numberDiff line numberDiff line change
@@ -4,18 +4,18 @@ import { parseLegacyLogsFrame } from './legacyLogsFrame';
44

55
// these are like Labels, but their values can be
66
// arbitrary structures, not just strings
7-
export type Attributes = Record<string, unknown>;
7+
export type LogFrameLabels = Record<string, unknown>;
88

99
// the attributes-access is a little awkward, but it's necessary
10-
// because there are multiple,very different dataframe-represenations.
10+
// because there are multiple,very different dataFrame-representations.
1111
export type LogsFrame = {
1212
timeField: FieldWithIndex;
1313
bodyField: FieldWithIndex;
1414
timeNanosecondField: FieldWithIndex | null;
1515
severityField: FieldWithIndex | null;
1616
idField: FieldWithIndex | null;
17-
getAttributes: () => Attributes[] | null; // may be slow, so we only do it when asked for it explicitly
18-
getAttributesAsLabels: () => Labels[] | null; // temporarily exists to make the labels=>attributes migration simpler
17+
getLogFrameLabels: () => LogFrameLabels[] | null; // may be slow, so we only do it when asked for it explicitly
18+
getLogFrameLabelsAsLabels: () => Labels[] | null; // temporarily exists to make the labels=>attributes migration simpler
1919
extraFields: FieldWithIndex[];
2020
};
2121

@@ -32,12 +32,12 @@ const DATAPLANE_TIMESTAMP_NAME = 'timestamp';
3232
const DATAPLANE_BODY_NAME = 'body';
3333
const DATAPLANE_SEVERITY_NAME = 'severity';
3434
const DATAPLANE_ID_NAME = 'id';
35-
const DATAPLANE_ATTRIBUTES_NAME = 'attributes';
35+
const DATAPLANE_LABELS_NAME = 'labels';
3636

37-
export function attributesToLabels(attributes: Attributes): Labels {
37+
export function logFrameLabelsToLabels(logFrameLabels: LogFrameLabels): Labels {
3838
const result: Labels = {};
3939

40-
Object.entries(attributes).forEach(([k, v]) => {
40+
Object.entries(logFrameLabels).forEach(([k, v]) => {
4141
result[k] = typeof v === 'string' ? v : JSON.stringify(v);
4242
});
4343

@@ -57,27 +57,27 @@ function parseDataplaneLogsFrame(frame: DataFrame): LogsFrame | null {
5757

5858
const severityField = getField(cache, DATAPLANE_SEVERITY_NAME, FieldType.string) ?? null;
5959
const idField = getField(cache, DATAPLANE_ID_NAME, FieldType.string) ?? null;
60-
const attributesField = getField(cache, DATAPLANE_ATTRIBUTES_NAME, FieldType.other) ?? null;
60+
const labelsField = getField(cache, DATAPLANE_LABELS_NAME, FieldType.other) ?? null;
6161

62-
const attributes = attributesField === null ? null : attributesField.values;
62+
const labels = labelsField === null ? null : labelsField.values;
6363

6464
const extraFields = cache.fields.filter(
6565
(_, i) =>
6666
i !== timestampField.index &&
6767
i !== bodyField.index &&
6868
i !== severityField?.index &&
6969
i !== idField?.index &&
70-
i !== attributesField?.index
70+
i !== labelsField?.index
7171
);
7272

7373
return {
7474
timeField: timestampField,
7575
bodyField,
7676
severityField,
7777
idField,
78-
getAttributes: () => attributes,
78+
getLogFrameLabels: () => labels,
7979
timeNanosecondField: null,
80-
getAttributesAsLabels: () => (attributes !== null ? attributes.map(attributesToLabels) : null),
80+
getLogFrameLabelsAsLabels: () => (labels !== null ? labels.map(logFrameLabelsToLabels) : null),
8181
extraFields,
8282
};
8383
}

‎public/app/features/logs/logsModel.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -366,7 +366,7 @@ export function logSeriesToLogsModel(logSeries: DataFrame[], queries: DataQuery[
366366
const logsFrame = parseLogsFrame(series);
367367
if (logsFrame != null) {
368368
// for now we ignore the nested-ness of attributes, and just stringify-them
369-
const frameLabels = logsFrame.getAttributesAsLabels() ?? undefined;
369+
const frameLabels = logsFrame.getLogFrameLabelsAsLabels() ?? undefined;
370370
const info = {
371371
rawFrame: series,
372372
logsFrame: logsFrame,

‎public/app/features/logs/logsModel_parse.test.ts

+2-2
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ jest.mock('@grafana/data', () => ({
1111
}));
1212

1313
describe('logSeriesToLogsModel should parse different logs-dataframe formats', () => {
14-
it('should parse a dataplane-formatted logs-frame)', () => {
14+
it('should parse a dataplane-formatted logs-frame', () => {
1515
const frames: DataFrame[] = [
1616
{
1717
refId: 'A',
@@ -23,7 +23,7 @@ describe('logSeriesToLogsModel should parse different logs-dataframe formats', (
2323
values: ['info', 'debug', 'error'],
2424
},
2525
{
26-
name: 'attributes',
26+
name: 'labels',
2727
type: FieldType.other,
2828
config: {},
2929
values: [

0 commit comments

Comments
 (0)