16
16
*
17
17
*/
18
18
19
+ use std:: collections:: HashMap ;
20
+ use std:: path:: Path ;
21
+
22
+ use crate :: about:: current;
19
23
use crate :: handlers:: http:: base_path_without_preceding_slash;
20
24
use crate :: handlers:: http:: ingest:: PostError ;
21
25
use crate :: handlers:: http:: modal:: IngestorMetadata ;
26
+ use crate :: option:: CONFIG ;
22
27
use crate :: utils:: get_url;
23
28
use crate :: HTTP_CLIENT ;
24
29
use actix_web:: http:: header;
@@ -33,6 +38,11 @@ use tracing::error;
33
38
use tracing:: warn;
34
39
use url:: Url ;
35
40
41
+ use super :: get_system_metrics;
42
+ use super :: get_volume_disk_usage;
43
+ use super :: DiskMetrics ;
44
+ use super :: MemoryMetrics ;
45
+
36
46
#[ derive( Debug , Serialize , Clone ) ]
37
47
pub struct Metrics {
38
48
address : String ,
@@ -51,6 +61,11 @@ pub struct Metrics {
51
61
event_time : NaiveDateTime ,
52
62
commit : String ,
53
63
staging : String ,
64
+ parseable_data_disk_usage : DiskMetrics ,
65
+ parseable_staging_disk_usage : DiskMetrics ,
66
+ parseable_hot_tier_disk_usage : DiskMetrics ,
67
+ parseable_memory_usage : MemoryMetrics ,
68
+ parseable_cpu_usage : HashMap < String , f64 > ,
54
69
}
55
70
56
71
#[ derive( Debug , Serialize , Default , Clone ) ]
@@ -59,15 +74,8 @@ struct StorageMetrics {
59
74
data : f64 ,
60
75
}
61
76
62
- impl Default for Metrics {
63
- fn default ( ) -> Self {
64
- let url = get_url ( ) ;
65
- let address = format ! (
66
- "http://{}:{}" ,
67
- url. domain( )
68
- . unwrap_or( url. host_str( ) . expect( "should have a host" ) ) ,
69
- url. port( ) . unwrap_or_default( )
70
- ) ;
77
+ impl Metrics {
78
+ fn new ( address : String ) -> Self {
71
79
Metrics {
72
80
address,
73
81
parseable_events_ingested : 0.0 ,
@@ -85,34 +93,240 @@ impl Default for Metrics {
85
93
event_time : Utc :: now ( ) . naive_utc ( ) ,
86
94
commit : "" . to_string ( ) ,
87
95
staging : "" . to_string ( ) ,
96
+ parseable_data_disk_usage : DiskMetrics {
97
+ total : 0 ,
98
+ used : 0 ,
99
+ available : 0 ,
100
+ } ,
101
+ parseable_staging_disk_usage : DiskMetrics {
102
+ total : 0 ,
103
+ used : 0 ,
104
+ available : 0 ,
105
+ } ,
106
+ parseable_hot_tier_disk_usage : DiskMetrics {
107
+ total : 0 ,
108
+ used : 0 ,
109
+ available : 0 ,
110
+ } ,
111
+ parseable_memory_usage : MemoryMetrics {
112
+ total : 0 ,
113
+ used : 0 ,
114
+ total_swap : 0 ,
115
+ used_swap : 0 ,
116
+ } ,
117
+ parseable_cpu_usage : HashMap :: new ( ) ,
88
118
}
89
119
}
90
120
}
91
121
92
- impl Metrics {
93
- fn new ( address : String ) -> Self {
94
- Metrics {
95
- address,
96
- parseable_events_ingested : 0.0 ,
97
- parseable_events_ingested_size : 0.0 ,
98
- parseable_staging_files : 0.0 ,
99
- process_resident_memory_bytes : 0.0 ,
100
- parseable_storage_size : StorageMetrics :: default ( ) ,
101
- parseable_lifetime_events_ingested : 0.0 ,
102
- parseable_lifetime_events_ingested_size : 0.0 ,
103
- parseable_deleted_events_ingested : 0.0 ,
104
- parseable_deleted_events_ingested_size : 0.0 ,
105
- parseable_deleted_storage_size : StorageMetrics :: default ( ) ,
106
- parseable_lifetime_storage_size : StorageMetrics :: default ( ) ,
107
- event_type : "cluster-metrics" . to_string ( ) ,
108
- event_time : Utc :: now ( ) . naive_utc ( ) ,
109
- commit : "" . to_string ( ) ,
110
- staging : "" . to_string ( ) ,
122
+ #[ derive( Debug ) ]
123
+ enum MetricType {
124
+ SimpleGauge ( String ) ,
125
+ StorageSize ( String ) ,
126
+ DiskUsage ( String ) ,
127
+ MemoryUsage ( String ) ,
128
+ CpuUsage ,
129
+ }
130
+
131
+ impl MetricType {
132
+ fn from_metric ( metric : & str , labels : & HashMap < String , String > ) -> Option < Self > {
133
+ match metric {
134
+ "parseable_events_ingested" => {
135
+ Some ( Self :: SimpleGauge ( "parseable_events_ingested" . into ( ) ) )
136
+ }
137
+ "parseable_events_ingested_size" => {
138
+ Some ( Self :: SimpleGauge ( "parseable_events_ingested_size" . into ( ) ) )
139
+ }
140
+ "parseable_lifetime_events_ingested" => Some ( Self :: SimpleGauge (
141
+ "parseable_lifetime_events_ingested" . into ( ) ,
142
+ ) ) ,
143
+ "parseable_lifetime_events_ingested_size" => Some ( Self :: SimpleGauge (
144
+ "parseable_lifetime_events_ingested_size" . into ( ) ,
145
+ ) ) ,
146
+ "parseable_events_deleted" => {
147
+ Some ( Self :: SimpleGauge ( "parseable_events_deleted" . into ( ) ) )
148
+ }
149
+ "parseable_events_deleted_size" => {
150
+ Some ( Self :: SimpleGauge ( "parseable_events_deleted_size" . into ( ) ) )
151
+ }
152
+ "parseable_staging_files" => Some ( Self :: SimpleGauge ( "parseable_staging_files" . into ( ) ) ) ,
153
+ "process_resident_memory_bytes" => {
154
+ Some ( Self :: SimpleGauge ( "process_resident_memory_bytes" . into ( ) ) )
155
+ }
156
+ "parseable_storage_size" => labels. get ( "type" ) . map ( |t| Self :: StorageSize ( t. clone ( ) ) ) ,
157
+ "parseable_lifetime_events_storage_size" => {
158
+ labels. get ( "type" ) . map ( |t| Self :: StorageSize ( t. clone ( ) ) )
159
+ }
160
+ "parseable_deleted_events_storage_size" => {
161
+ labels. get ( "type" ) . map ( |t| Self :: StorageSize ( t. clone ( ) ) )
162
+ }
163
+ "parseable_total_disk" | "parseable_used_disk" | "parseable_available_disk" => {
164
+ labels. get ( "volume" ) . map ( |v| Self :: DiskUsage ( v. clone ( ) ) )
165
+ }
166
+ "parseable_memory_usage" => labels
167
+ . get ( "memory_usage" )
168
+ . map ( |m| Self :: MemoryUsage ( m. clone ( ) ) ) ,
169
+ "parseable_cpu_usage" => Some ( Self :: CpuUsage ) ,
170
+ _ => None ,
111
171
}
112
172
}
113
173
}
114
-
115
174
impl Metrics {
175
+ pub async fn ingestor_prometheus_samples (
176
+ samples : Vec < PromSample > ,
177
+ ingestor_metadata : & IngestorMetadata ,
178
+ ) -> Result < Self , PostError > {
179
+ let mut metrics = Metrics :: new ( ingestor_metadata. domain_name . to_string ( ) ) ;
180
+
181
+ Self :: build_metrics_from_samples ( samples, & mut metrics) ?;
182
+
183
+ // Get additional metadata
184
+ let ( commit_id, staging) = Self :: from_about_api_response ( ingestor_metadata. clone ( ) )
185
+ . await
186
+ . map_err ( |err| {
187
+ error ! ( "Fatal: failed to get ingestor info: {:?}" , err) ;
188
+ PostError :: Invalid ( err. into ( ) )
189
+ } ) ?;
190
+
191
+ metrics. commit = commit_id;
192
+ metrics. staging = staging;
193
+
194
+ Ok ( metrics)
195
+ }
196
+
197
+ pub async fn querier_prometheus_metrics ( ) -> Self {
198
+ let mut metrics = Metrics :: new ( get_url ( ) . to_string ( ) ) ;
199
+
200
+ let system_metrics = get_system_metrics ( ) . expect ( "Failed to get system metrics" ) ;
201
+
202
+ metrics. parseable_memory_usage . total = system_metrics. memory . total ;
203
+ metrics. parseable_memory_usage . used = system_metrics. memory . used ;
204
+ metrics. parseable_memory_usage . total_swap = system_metrics. memory . total_swap ;
205
+ metrics. parseable_memory_usage . used_swap = system_metrics. memory . used_swap ;
206
+ for cpu_usage in system_metrics. cpu {
207
+ metrics
208
+ . parseable_cpu_usage
209
+ . insert ( cpu_usage. name . clone ( ) , cpu_usage. usage ) ;
210
+ }
211
+
212
+ let staging_disk_usage = get_volume_disk_usage ( CONFIG . staging_dir ( ) )
213
+ . expect ( "Failed to get staging volume disk usage" ) ;
214
+
215
+ metrics. parseable_staging_disk_usage . total = staging_disk_usage. total ;
216
+ metrics. parseable_staging_disk_usage . used = staging_disk_usage. used ;
217
+ metrics. parseable_staging_disk_usage . available = staging_disk_usage. available ;
218
+
219
+ if CONFIG . get_storage_mode_string ( ) == "Local drive" {
220
+ let data_disk_usage =
221
+ get_volume_disk_usage ( Path :: new ( & CONFIG . storage ( ) . get_endpoint ( ) ) )
222
+ . expect ( "Failed to get data volume disk usage" ) ;
223
+
224
+ metrics. parseable_data_disk_usage . total = data_disk_usage. total ;
225
+ metrics. parseable_data_disk_usage . used = data_disk_usage. used ;
226
+ metrics. parseable_data_disk_usage . available = data_disk_usage. available ;
227
+ }
228
+
229
+ if CONFIG . options . hot_tier_storage_path . is_some ( ) {
230
+ let hot_tier_disk_usage =
231
+ get_volume_disk_usage ( CONFIG . hot_tier_dir ( ) . as_ref ( ) . unwrap ( ) )
232
+ . expect ( "Failed to get hot tier volume disk usage" ) ;
233
+
234
+ metrics. parseable_hot_tier_disk_usage . total = hot_tier_disk_usage. total ;
235
+ metrics. parseable_hot_tier_disk_usage . used = hot_tier_disk_usage. used ;
236
+ metrics. parseable_hot_tier_disk_usage . available = hot_tier_disk_usage. available ;
237
+ }
238
+
239
+ metrics. commit = current ( ) . commit_hash ;
240
+ metrics. staging = CONFIG . staging_dir ( ) . display ( ) . to_string ( ) ;
241
+
242
+ metrics
243
+ }
244
+
245
+ fn build_metrics_from_samples (
246
+ samples : Vec < PromSample > ,
247
+ metrics : & mut Metrics ,
248
+ ) -> Result < ( ) , PostError > {
249
+ for sample in samples {
250
+ let metric_type = MetricType :: from_metric ( & sample. metric , & sample. labels ) ;
251
+
252
+ match ( sample. value . clone ( ) , metric_type) {
253
+ ( PromValue :: Gauge ( val) , Some ( metric_type) ) => {
254
+ Self :: process_gauge_metric (
255
+ metrics,
256
+ metric_type,
257
+ val,
258
+ & sample. metric ,
259
+ sample. clone ( ) ,
260
+ ) ;
261
+ }
262
+ _ => continue ,
263
+ }
264
+ }
265
+ Ok ( ( ) )
266
+ }
267
+
268
+ fn process_gauge_metric (
269
+ metrics : & mut Metrics ,
270
+ metric_type : MetricType ,
271
+ val : f64 ,
272
+ metric_name : & str ,
273
+ sample : PromSample ,
274
+ ) {
275
+ match metric_type {
276
+ MetricType :: SimpleGauge ( metric_name) => match metric_name. as_str ( ) {
277
+ "parseable_events_ingested" => metrics. parseable_events_ingested += val,
278
+ "parseable_events_ingested_size" => metrics. parseable_events_ingested_size += val,
279
+ "parseable_lifetime_events_ingested" => {
280
+ metrics. parseable_lifetime_events_ingested += val
281
+ }
282
+ "parseable_lifetime_events_ingested_size" => {
283
+ metrics. parseable_lifetime_events_ingested_size += val
284
+ }
285
+ "parseable_events_deleted" => metrics. parseable_deleted_events_ingested += val,
286
+ "parseable_events_deleted_size" => {
287
+ metrics. parseable_deleted_events_ingested_size += val
288
+ }
289
+ "parseable_staging_files" => metrics. parseable_staging_files += val,
290
+ "process_resident_memory_bytes" => metrics. process_resident_memory_bytes += val,
291
+ _ => { }
292
+ } ,
293
+ MetricType :: StorageSize ( storage_type) => match storage_type. as_str ( ) {
294
+ "staging" => metrics. parseable_storage_size . staging += val,
295
+ "data" => metrics. parseable_storage_size . data += val,
296
+ _ => { }
297
+ } ,
298
+ MetricType :: DiskUsage ( volume_type) => {
299
+ let disk_usage = match volume_type. as_str ( ) {
300
+ "data" => & mut metrics. parseable_data_disk_usage ,
301
+ "staging" => & mut metrics. parseable_staging_disk_usage ,
302
+ "hot_tier" => & mut metrics. parseable_hot_tier_disk_usage ,
303
+ _ => return ,
304
+ } ;
305
+
306
+ match metric_name {
307
+ "parseable_total_disk" => disk_usage. total = val as u64 ,
308
+ "parseable_used_disk" => disk_usage. used = val as u64 ,
309
+ "parseable_available_disk" => disk_usage. available = val as u64 ,
310
+ _ => { }
311
+ }
312
+ }
313
+ MetricType :: MemoryUsage ( memory_type) => match memory_type. as_str ( ) {
314
+ "total_memory" => metrics. parseable_memory_usage . total = val as u64 ,
315
+ "used_memory" => metrics. parseable_memory_usage . used = val as u64 ,
316
+ "total_swap" => metrics. parseable_memory_usage . total_swap = val as u64 ,
317
+ "used_swap" => metrics. parseable_memory_usage . used_swap = val as u64 ,
318
+ _ => { }
319
+ } ,
320
+ MetricType :: CpuUsage => {
321
+ if let Some ( cpu_name) = sample. labels . get ( "cpu_usage" ) {
322
+ metrics
323
+ . parseable_cpu_usage
324
+ . insert ( cpu_name. to_string ( ) , val) ;
325
+ }
326
+ }
327
+ }
328
+ }
329
+
116
330
pub fn get_daily_stats_from_samples (
117
331
samples : Vec < PromSample > ,
118
332
stream_name : & str ,
@@ -154,68 +368,6 @@ impl Metrics {
154
368
}
155
369
( events_ingested, ingestion_size, storage_size)
156
370
}
157
- pub async fn from_prometheus_samples (
158
- samples : Vec < PromSample > ,
159
- ingestor_metadata : & IngestorMetadata ,
160
- ) -> Result < Self , PostError > {
161
- let mut prom_dress = Metrics :: new ( ingestor_metadata. domain_name . to_string ( ) ) ;
162
- for sample in samples {
163
- if let PromValue :: Gauge ( val) = sample. value {
164
- match sample. metric . as_str ( ) {
165
- "parseable_events_ingested" => prom_dress. parseable_events_ingested += val,
166
- "parseable_events_ingested_size" => {
167
- prom_dress. parseable_events_ingested_size += val
168
- }
169
- "parseable_lifetime_events_ingested" => {
170
- prom_dress. parseable_lifetime_events_ingested += val
171
- }
172
- "parseable_lifetime_events_ingested_size" => {
173
- prom_dress. parseable_lifetime_events_ingested_size += val
174
- }
175
- "parseable_events_deleted" => {
176
- prom_dress. parseable_deleted_events_ingested += val
177
- }
178
- "parseable_events_deleted_size" => {
179
- prom_dress. parseable_deleted_events_ingested_size += val
180
- }
181
- "parseable_staging_files" => prom_dress. parseable_staging_files += val,
182
- "process_resident_memory_bytes" => {
183
- prom_dress. process_resident_memory_bytes += val
184
- }
185
- "parseable_storage_size" => {
186
- if sample. labels . get ( "type" ) . expect ( "type is present" ) == "staging" {
187
- prom_dress. parseable_storage_size . staging += val;
188
- }
189
- if sample. labels . get ( "type" ) . expect ( "type is present" ) == "data" {
190
- prom_dress. parseable_storage_size . data += val;
191
- }
192
- }
193
- "parseable_lifetime_events_storage_size" => {
194
- if sample. labels . get ( "type" ) . expect ( "type is present" ) == "data" {
195
- prom_dress. parseable_lifetime_storage_size . data += val;
196
- }
197
- }
198
- "parseable_deleted_events_storage_size" => {
199
- if sample. labels . get ( "type" ) . expect ( "type is present" ) == "data" {
200
- prom_dress. parseable_deleted_storage_size . data += val;
201
- }
202
- }
203
- _ => { }
204
- }
205
- }
206
- }
207
- let ( commit_id, staging) = Self :: from_about_api_response ( ingestor_metadata. clone ( ) )
208
- . await
209
- . map_err ( |err| {
210
- error ! ( "Fatal: failed to get ingestor info: {:?}" , err) ;
211
- PostError :: Invalid ( err. into ( ) )
212
- } ) ?;
213
-
214
- prom_dress. commit = commit_id;
215
- prom_dress. staging = staging;
216
-
217
- Ok ( prom_dress)
218
- }
219
371
220
372
pub async fn from_about_api_response (
221
373
ingestor_metadata : IngestorMetadata ,
0 commit comments