feat: add sysinfo metrics

collect CPU usage, memory usage of the server
collect disk usage of the volume - data, staging, hot-tier
add these metrics to Prometheus Metrics

export these metrics to cluster metrics API
add the metrics to pmeta stream
add the querier node's sysinfo metrics to pmeta and cluster metrics API

Author: nikhilsinhaparseable

src/correlation.rs

@@ -129,10 +129,9 @@ impl Correlations {
             .await?;
 
         // Update in memory
-        self.write().await.insert(
-            correlation.id.to_owned(),
-            correlation.clone(),
-        );
+        self.write()
+            .await
+            .insert(correlation.id.to_owned(), correlation.clone());
 
         Ok(correlation)
     }

src/handlers/http/cluster/mod.rs

@@ -35,7 +35,7 @@ use crate::storage::{ObjectStorageError, STREAM_ROOT_DIRECTORY};
 use crate::storage::{ObjectStoreFormat, PARSEABLE_ROOT_DIRECTORY};
 use crate::HTTP_CLIENT;
 use actix_web::http::header::{self, HeaderMap};
-use actix_web::web::Path;
+use actix_web::web::{Json, Path};
 use actix_web::Responder;
 use bytes::Bytes;
 use chrono::Utc;
@@ -729,6 +729,9 @@ async fn fetch_cluster_metrics() -> Result<Vec<Metrics>, PostError> {
     let mut dresses = vec![];
 
     for ingestor in ingestor_metadata {
+        if !utils::check_liveness(&ingestor.domain_name).await {
+            continue;
+        }
         let uri = Url::parse(&format!(
             "{}{}/metrics",
             &ingestor.domain_name,
@@ -749,11 +752,10 @@ async fn fetch_cluster_metrics() -> Result<Vec<Metrics>, PostError> {
             let text = res.text().await.map_err(PostError::NetworkError)?;
             let lines: Vec<Result<String, std::io::Error>> =
                 text.lines().map(|line| Ok(line.to_owned())).collect_vec();
-
             let sample = prometheus_parse::Scrape::parse(lines.into_iter())
                 .map_err(|err| PostError::CustomError(err.to_string()))?
                 .samples;
-            let ingestor_metrics = Metrics::from_prometheus_samples(sample, &ingestor)
+            let ingestor_metrics = Metrics::ingestor_prometheus_samples(sample, &ingestor)
                 .await
                 .map_err(|err| {
                     error!("Fatal: failed to get ingestor metrics: {:?}", err);
@@ -767,10 +769,11 @@ async fn fetch_cluster_metrics() -> Result<Vec<Metrics>, PostError> {
             );
         }
     }
+    dresses.push(Metrics::querier_prometheus_metrics().await);
     Ok(dresses)
 }
 
-pub fn init_cluster_metrics_schedular() -> Result<(), PostError> {
+pub async fn init_cluster_metrics_scheduler() -> Result<(), PostError> {
     info!("Setting up schedular for cluster metrics ingestion");
     let mut scheduler = AsyncScheduler::new();
     scheduler
@@ -779,25 +782,12 @@ pub fn init_cluster_metrics_schedular() -> Result<(), PostError> {
             let result: Result<(), PostError> = async {
                 let cluster_metrics = fetch_cluster_metrics().await;
                 if let Ok(metrics) = cluster_metrics {
-                    if !metrics.is_empty() {
-                        info!("Cluster metrics fetched successfully from all ingestors");
-                        if let Ok(metrics_bytes) = serde_json::to_vec(&metrics) {
-                            if matches!(
-                                ingest_internal_stream(
-                                    INTERNAL_STREAM_NAME.to_string(),
-                                    bytes::Bytes::from(metrics_bytes),
-                                )
-                                .await,
-                                Ok(())
-                            ) {
-                                info!("Cluster metrics successfully ingested into internal stream");
-                            } else {
-                                error!("Failed to ingest cluster metrics into internal stream");
-                            }
-                        } else {
-                            error!("Failed to serialize cluster metrics");
-                        }
-                    }
+                    let json_value = serde_json::to_value(metrics)
+                        .map_err(|e| anyhow::anyhow!("Failed to serialize metrics: {}", e))?;
+
+                    ingest_internal_stream(INTERNAL_STREAM_NAME.to_string(), Json(json_value))
+                        .await
+                        .map_err(|e| anyhow::anyhow!("Failed to ingest metrics: {}", e))?;
                 }
                 Ok(())
             }

src/handlers/http/ingest.rs

@@ -20,12 +20,8 @@ use super::logstream::error::{CreateStreamError, StreamError};
 use super::modal::utils::ingest_utils::{flatten_and_push_logs, push_logs};
 use super::users::dashboards::DashboardError;
 use super::users::filters::FiltersError;
-use crate::event::format::LogSource;
-use crate::event::{
-    self,
-    error::EventError,
-    format::{self, EventFormat},
-};
+use crate::event::format::{self, EventFormat, LogSource};
+use crate::event::{self, error::EventError};
 use crate::handlers::http::modal::utils::logstream_utils::create_stream_and_schema_from_storage;
 use crate::handlers::{LOG_SOURCE_KEY, STREAM_NAME_HEADER_KEY};
 use crate::metadata::error::stream_info::MetadataError;
@@ -36,12 +32,12 @@ use crate::otel::metrics::flatten_otel_metrics;
 use crate::otel::traces::flatten_otel_traces;
 use crate::storage::{ObjectStorageError, StreamType};
 use crate::utils::header_parsing::ParseHeaderError;
-use crate::utils::json::flatten::JsonFlattenError;
+use crate::utils::json::convert_array_to_object;
+use crate::utils::json::flatten::{convert_to_array, JsonFlattenError};
 use actix_web::web::{Json, Path};
 use actix_web::{http::header::ContentType, HttpRequest, HttpResponse};
 use arrow_array::RecordBatch;
 use arrow_schema::Schema;
-use bytes::Bytes;
 use chrono::Utc;
 use http::StatusCode;
 use opentelemetry_proto::tonic::logs::v1::LogsData;
@@ -77,18 +73,29 @@ pub async fn ingest(req: HttpRequest, Json(json): Json<Value>) -> Result<HttpRes
     Ok(HttpResponse::Ok().finish())
 }
 
-pub async fn ingest_internal_stream(stream_name: String, body: Bytes) -> Result<(), PostError> {
-    let size: usize = body.len();
+pub async fn ingest_internal_stream(
+    stream_name: String,
+    Json(json): Json<Value>,
+) -> Result<(), PostError> {
+    let size = serde_json::to_vec(&json).unwrap().len() as u64;
+    let data = convert_to_array(convert_array_to_object(
+        json,
+        None,
+        None,
+        None,
+        SchemaVersion::V0,
+        &LogSource::default(),
+    )?)?;
+
     let parsed_timestamp = Utc::now().naive_utc();
     let (rb, is_first) = {
-        let body_val: Value = serde_json::from_slice(&body)?;
         let hash_map = STREAM_INFO.read().unwrap();
         let schema = hash_map
             .get(&stream_name)
             .ok_or(PostError::StreamNotFound(stream_name.clone()))?
             .schema
             .clone();
-        let event = format::json::Event { data: body_val };
+        let event = format::json::Event { data };
         // For internal streams, use old schema
         event.into_recordbatch(&schema, false, None, SchemaVersion::V0)?
     };

src/handlers/http/modal/ingest_server.rs

@@ -113,6 +113,7 @@ impl ParseableServer for IngestServer {
 
         // set the ingestor metadata
         set_ingestor_metadata().await?;
+        metrics::init_system_metrics_scheduler().await?;
 
         // Ingestors shouldn't have to deal with OpenId auth flow
         let app = self.start(shutdown_rx, prometheus, None);

src/handlers/http/modal/query_server.rs

@@ -18,11 +18,10 @@
 
 use crate::correlation::CORRELATIONS;
 use crate::handlers::airplane;
-use crate::handlers::http::base_path;
-use crate::handlers::http::cluster::{self, init_cluster_metrics_schedular};
 use crate::handlers::http::logstream::create_internal_stream_if_not_exists;
 use crate::handlers::http::middleware::{DisAllowRootUser, RouteExt};
 use crate::handlers::http::{self, role};
+use crate::handlers::http::{base_path, cluster};
 use crate::handlers::http::{logstream, MAX_EVENT_PAYLOAD_SIZE};
 use crate::hottier::HotTierManager;
 use crate::rbac::role::Action;
@@ -35,7 +34,7 @@ use actix_web::{web, Scope};
 use async_trait::async_trait;
 use bytes::Bytes;
 use tokio::sync::oneshot;
-use tracing::{error, info};
+use tracing::error;
 
 use crate::{option::CONFIG, ParseableServer};
 
@@ -104,15 +103,14 @@ impl ParseableServer for QueryServer {
         // track all parquet files already in the data directory
         storage::retention::load_retention_from_global();
 
+        metrics::init_system_metrics_scheduler().await?;
+        cluster::init_cluster_metrics_scheduler().await?;
         // all internal data structures populated now.
         // start the analytics scheduler if enabled
         if CONFIG.options.send_analytics {
             analytics::init_analytics_scheduler()?;
         }
 
-        if matches!(init_cluster_metrics_schedular(), Ok(())) {
-            info!("Cluster metrics scheduler started successfully");
-        }
         if let Some(hot_tier_manager) = HotTierManager::global() {
             hot_tier_manager.put_internal_stream_hot_tier().await?;
             hot_tier_manager.download_from_s3()?;

src/handlers/http/modal/server.rs

@@ -119,6 +119,8 @@ impl ParseableServer for Server {
         let (mut remote_sync_handler, mut remote_sync_outbox, mut remote_sync_inbox) =
             sync::object_store_sync().await;
 
+        metrics::init_system_metrics_scheduler().await?;
+
         if CONFIG.options.send_analytics {
             analytics::init_analytics_scheduler()?;
         }

src/metrics/mod.rs

@@ -18,15 +18,25 @@
 
 pub mod prom_utils;
 pub mod storage;
-
-use crate::{handlers::http::metrics_path, stats::FullStats};
+use actix_web::HttpResponse;
+use clokwerk::{AsyncScheduler, Interval};
+use http::StatusCode;
+use serde::Serialize;
+use std::{path::Path, time::Duration};
+use sysinfo::{Disks, System};
+use tracing::{error, info};
+
+use crate::{handlers::http::metrics_path, option::CONFIG, stats::FullStats};
 use actix_web::Responder;
 use actix_web_prometheus::{PrometheusMetrics, PrometheusMetricsBuilder};
 use error::MetricsError;
 use once_cell::sync::Lazy;
-use prometheus::{HistogramOpts, HistogramVec, IntCounterVec, IntGaugeVec, Opts, Registry};
+use prometheus::{
+    GaugeVec, HistogramOpts, HistogramVec, IntCounterVec, IntGaugeVec, Opts, Registry,
+};
 
 pub const METRICS_NAMESPACE: &str = env!("CARGO_PKG_NAME");
+const SYSTEM_METRICS_INTERVAL_SECONDS: Interval = clokwerk::Interval::Minutes(1);
 
 pub static EVENTS_INGESTED: Lazy<IntGaugeVec> = Lazy::new(|| {
     IntGaugeVec::new(
@@ -182,6 +192,42 @@ pub static ALERTS_STATES: Lazy<IntCounterVec> = Lazy::new(|| {
     .expect("metric can be created")
 });
 
+pub static TOTAL_DISK: Lazy<IntGaugeVec> = Lazy::new(|| {
+    IntGaugeVec::new(
+        Opts::new("total_disk", "Total Disk Size").namespace(METRICS_NAMESPACE),
+        &["volume"],
+    )
+    .expect("metric can be created")
+});
+pub static USED_DISK: Lazy<IntGaugeVec> = Lazy::new(|| {
+    IntGaugeVec::new(
+        Opts::new("used_disk", "Used Disk Size").namespace(METRICS_NAMESPACE),
+        &["volume"],
+    )
+    .expect("metric can be created")
+});
+pub static AVAILABLE_DISK: Lazy<IntGaugeVec> = Lazy::new(|| {
+    IntGaugeVec::new(
+        Opts::new("available_disk", "Available Disk Size").namespace(METRICS_NAMESPACE),
+        &["volume"],
+    )
+    .expect("metric can be created")
+});
+pub static MEMORY: Lazy<IntGaugeVec> = Lazy::new(|| {
+    IntGaugeVec::new(
+        Opts::new("memory_usage", "Memory Usage").namespace(METRICS_NAMESPACE),
+        &["memory_usage"],
+    )
+    .expect("metric can be created")
+});
+pub static CPU: Lazy<GaugeVec> = Lazy::new(|| {
+    GaugeVec::new(
+        Opts::new("cpu_usage", "CPU Usage").namespace(METRICS_NAMESPACE),
+        &["cpu_usage"],
+    )
+    .expect("metric can be created")
+});
+
 fn custom_metrics(registry: &Registry) {
     registry
         .register(Box::new(EVENTS_INGESTED.clone()))
@@ -231,6 +277,21 @@ fn custom_metrics(registry: &Registry) {
     registry
         .register(Box::new(ALERTS_STATES.clone()))
         .expect("metric can be registered");
+    registry
+        .register(Box::new(TOTAL_DISK.clone()))
+        .expect("metric can be registered");
+    registry
+        .register(Box::new(USED_DISK.clone()))
+        .expect("metric can be registered");
+    registry
+        .register(Box::new(AVAILABLE_DISK.clone()))
+        .expect("metric can be registered");
+    registry
+        .register(Box::new(MEMORY.clone()))
+        .expect("metric can be registered");
+    registry
+        .register(Box::new(CPU.clone()))
+        .expect("metric can be registered");
 }
 
 pub fn build_metrics_handler() -> PrometheusMetrics {
@@ -290,12 +351,196 @@ pub async fn fetch_stats_from_storage(stream_name: &str, stats: FullStats) {
         .set(stats.lifetime_stats.storage as i64);
 }
 
-use actix_web::HttpResponse;
-
 pub async fn get() -> Result<impl Responder, MetricsError> {
     Ok(HttpResponse::Ok().body(format!("{:?}", build_metrics_handler())))
 }
 
+#[derive(Debug, Serialize, Default, Clone)]
+pub struct DiskMetrics {
+    total: u64,
+    used: u64,
+    available: u64,
+}
+
+#[derive(Debug, Serialize, Default, Clone)]
+pub struct SystemMetrics {
+    memory: MemoryMetrics,
+    cpu: Vec<CpuMetrics>,
+}
+
+#[derive(Debug, Serialize, Default, Clone)]
+pub struct MemoryMetrics {
+    total: u64,
+    used: u64,
+    total_swap: u64,
+    used_swap: u64,
+}
+
+#[derive(Debug, Serialize, Default, Clone)]
+pub struct CpuMetrics {
+    name: String,
+    usage: f64,
+}
+
+// Scheduler for collecting all system metrics
+pub async fn init_system_metrics_scheduler() -> Result<(), MetricsError> {
+    info!("Setting up scheduler for capturing system metrics");
+    let mut scheduler = AsyncScheduler::new();
+
+    scheduler
+        .every(SYSTEM_METRICS_INTERVAL_SECONDS)
+        .run(move || async {
+            if let Err(err) = collect_all_metrics().await {
+                error!("Error in capturing system metrics: {:#}", err);
+            }
+        });
+
+    tokio::spawn(async move {
+        loop {
+            scheduler.run_pending().await;
+            tokio::time::sleep(Duration::from_secs(10)).await;
+        }
+    });
+
+    Ok(())
+}
+
+// Function to collect memory, CPU and disk usage metrics
+pub async fn collect_all_metrics() -> Result<(), MetricsError> {
+    // Collect system metrics (CPU and memory)
+    collect_system_metrics().await?;
+
+    // Collect disk metrics for all volumes
+    collect_disk_metrics().await?;
+
+    Ok(())
+}
+
+// Function to collect disk usage metrics
+async fn collect_disk_metrics() -> Result<(), MetricsError> {
+    // collect staging volume metrics
+    collect_volume_disk_usage("staging", CONFIG.staging_dir())?;
+    // Collect data volume metrics for local storage
+    if CONFIG.get_storage_mode_string() == "Local drive" {
+        collect_volume_disk_usage("data", Path::new(&CONFIG.storage().get_endpoint()))?;
+    }
+
+    // Collect hot tier volume metrics if configured
+    if let Some(hot_tier_dir) = CONFIG.hot_tier_dir() {
+        collect_volume_disk_usage("hot_tier", hot_tier_dir)?;
+    }
+
+    Ok(())
+}
+
+// Function to collect disk usage metrics for a specific volume
+fn collect_volume_disk_usage(label: &str, path: &Path) -> Result<(), MetricsError> {
+    let metrics = get_volume_disk_usage(path)?;
+
+    TOTAL_DISK
+        .with_label_values(&[label])
+        .set(metrics.total as i64);
+    USED_DISK
+        .with_label_values(&[label])
+        .set(metrics.used as i64);
+    AVAILABLE_DISK
+        .with_label_values(&[label])
+        .set(metrics.available as i64);
+
+    Ok(())
+}
+
+// Function to get disk usage for a specific volume
+fn get_volume_disk_usage(path: &Path) -> Result<DiskMetrics, MetricsError> {
+    let mut disks = Disks::new_with_refreshed_list();
+    disks.sort_by(|a, b| {
+        b.mount_point()
+            .to_str()
+            .unwrap_or("")
+            .len()
+            .cmp(&a.mount_point().to_str().unwrap_or("").len())
+    });
+
+    for disk in disks.iter() {
+        let mount_point = disk.mount_point().to_str().unwrap();
+
+        if path.starts_with(mount_point) {
+            return Ok(DiskMetrics {
+                total: disk.total_space(),
+                used: disk.total_space() - disk.available_space(),
+                available: disk.available_space(),
+            });
+        }
+    }
+
+    Err(MetricsError::Custom(
+        format!("No matching disk found for path: {:?}", path),
+        StatusCode::INTERNAL_SERVER_ERROR,
+    ))
+}
+
+// Function to collect CPU and memory usage metrics
+async fn collect_system_metrics() -> Result<(), MetricsError> {
+    let metrics = get_system_metrics()?;
+
+    // Set memory metrics
+    MEMORY
+        .with_label_values(&["total_memory"])
+        .set(metrics.memory.total as i64);
+    MEMORY
+        .with_label_values(&["used_memory"])
+        .set(metrics.memory.used as i64);
+    MEMORY
+        .with_label_values(&["total_swap"])
+        .set(metrics.memory.total_swap as i64);
+    MEMORY
+        .with_label_values(&["used_swap"])
+        .set(metrics.memory.used_swap as i64);
+
+    // Set CPU metrics
+    for cpu in metrics.cpu {
+        CPU.with_label_values(&[&cpu.name]).set(cpu.usage);
+    }
+
+    Ok(())
+}
+
+// Get system metrics
+fn get_system_metrics() -> Result<SystemMetrics, MetricsError> {
+    let mut sys = System::new_all();
+    sys.refresh_all();
+
+    // Collect memory metrics
+    let memory = MemoryMetrics {
+        total: sys.total_memory(),
+        used: sys.used_memory(),
+        total_swap: sys.total_swap(),
+        used_swap: sys.used_swap(),
+    };
+
+    // Collect CPU metrics
+    let mut cpu_metrics = Vec::new();
+
+    // Add global CPU usage
+    cpu_metrics.push(CpuMetrics {
+        name: "global".to_string(),
+        usage: sys.global_cpu_usage() as f64,
+    });
+
+    // Add individual CPU usage
+    for cpu in sys.cpus() {
+        cpu_metrics.push(CpuMetrics {
+            name: cpu.name().to_string(),
+            usage: cpu.cpu_usage() as f64,
+        });
+    }
+
+    Ok(SystemMetrics {
+        memory,
+        cpu: cpu_metrics,
+    })
+}
+
 pub mod error {
 
     use actix_web::http::header::ContentType;

src/metrics/prom_utils.rs

@@ -16,9 +16,14 @@
  *
  */
 
+use std::collections::HashMap;
+use std::path::Path;
+
+use crate::about::current;
 use crate::handlers::http::base_path_without_preceding_slash;
 use crate::handlers::http::ingest::PostError;
 use crate::handlers::http::modal::IngestorMetadata;
+use crate::option::CONFIG;
 use crate::utils::get_url;
 use crate::HTTP_CLIENT;
 use actix_web::http::header;
@@ -33,6 +38,11 @@ use tracing::error;
 use tracing::warn;
 use url::Url;
 
+use super::get_system_metrics;
+use super::get_volume_disk_usage;
+use super::DiskMetrics;
+use super::MemoryMetrics;
+
 #[derive(Debug, Serialize, Clone)]
 pub struct Metrics {
     address: String,
@@ -51,6 +61,11 @@ pub struct Metrics {
     event_time: NaiveDateTime,
     commit: String,
     staging: String,
+    parseable_data_disk_usage: DiskMetrics,
+    parseable_staging_disk_usage: DiskMetrics,
+    parseable_hot_tier_disk_usage: DiskMetrics,
+    parseable_memory_usage: MemoryMetrics,
+    parseable_cpu_usage: HashMap<String, f64>,
 }
 
 #[derive(Debug, Serialize, Default, Clone)]
@@ -59,15 +74,8 @@ struct StorageMetrics {
     data: f64,
 }
 
-impl Default for Metrics {
-    fn default() -> Self {
-        let url = get_url();
-        let address = format!(
-            "http://{}:{}",
-            url.domain()
-                .unwrap_or(url.host_str().expect("should have a host")),
-            url.port().unwrap_or_default()
-        );
+impl Metrics {
+    fn new(address: String) -> Self {
         Metrics {
             address,
             parseable_events_ingested: 0.0,
@@ -85,34 +93,240 @@ impl Default for Metrics {
             event_time: Utc::now().naive_utc(),
             commit: "".to_string(),
             staging: "".to_string(),
+            parseable_data_disk_usage: DiskMetrics {
+                total: 0,
+                used: 0,
+                available: 0,
+            },
+            parseable_staging_disk_usage: DiskMetrics {
+                total: 0,
+                used: 0,
+                available: 0,
+            },
+            parseable_hot_tier_disk_usage: DiskMetrics {
+                total: 0,
+                used: 0,
+                available: 0,
+            },
+            parseable_memory_usage: MemoryMetrics {
+                total: 0,
+                used: 0,
+                total_swap: 0,
+                used_swap: 0,
+            },
+            parseable_cpu_usage: HashMap::new(),
         }
     }
 }
 
-impl Metrics {
-    fn new(address: String) -> Self {
-        Metrics {
-            address,
-            parseable_events_ingested: 0.0,
-            parseable_events_ingested_size: 0.0,
-            parseable_staging_files: 0.0,
-            process_resident_memory_bytes: 0.0,
-            parseable_storage_size: StorageMetrics::default(),
-            parseable_lifetime_events_ingested: 0.0,
-            parseable_lifetime_events_ingested_size: 0.0,
-            parseable_deleted_events_ingested: 0.0,
-            parseable_deleted_events_ingested_size: 0.0,
-            parseable_deleted_storage_size: StorageMetrics::default(),
-            parseable_lifetime_storage_size: StorageMetrics::default(),
-            event_type: "cluster-metrics".to_string(),
-            event_time: Utc::now().naive_utc(),
-            commit: "".to_string(),
-            staging: "".to_string(),
+#[derive(Debug)]
+enum MetricType {
+    SimpleGauge(String),
+    StorageSize(String),
+    DiskUsage(String),
+    MemoryUsage(String),
+    CpuUsage,
+}
+
+impl MetricType {
+    fn from_metric(metric: &str, labels: &HashMap<String, String>) -> Option<Self> {
+        match metric {
+            "parseable_events_ingested" => {
+                Some(Self::SimpleGauge("parseable_events_ingested".into()))
+            }
+            "parseable_events_ingested_size" => {
+                Some(Self::SimpleGauge("parseable_events_ingested_size".into()))
+            }
+            "parseable_lifetime_events_ingested" => Some(Self::SimpleGauge(
+                "parseable_lifetime_events_ingested".into(),
+            )),
+            "parseable_lifetime_events_ingested_size" => Some(Self::SimpleGauge(
+                "parseable_lifetime_events_ingested_size".into(),
+            )),
+            "parseable_events_deleted" => {
+                Some(Self::SimpleGauge("parseable_events_deleted".into()))
+            }
+            "parseable_events_deleted_size" => {
+                Some(Self::SimpleGauge("parseable_events_deleted_size".into()))
+            }
+            "parseable_staging_files" => Some(Self::SimpleGauge("parseable_staging_files".into())),
+            "process_resident_memory_bytes" => {
+                Some(Self::SimpleGauge("process_resident_memory_bytes".into()))
+            }
+            "parseable_storage_size" => labels.get("type").map(|t| Self::StorageSize(t.clone())),
+            "parseable_lifetime_events_storage_size" => {
+                labels.get("type").map(|t| Self::StorageSize(t.clone()))
+            }
+            "parseable_deleted_events_storage_size" => {
+                labels.get("type").map(|t| Self::StorageSize(t.clone()))
+            }
+            "parseable_total_disk" | "parseable_used_disk" | "parseable_available_disk" => {
+                labels.get("volume").map(|v| Self::DiskUsage(v.clone()))
+            }
+            "parseable_memory_usage" => labels
+                .get("memory_usage")
+                .map(|m| Self::MemoryUsage(m.clone())),
+            "parseable_cpu_usage" => Some(Self::CpuUsage),
+            _ => None,
         }
     }
 }
-
 impl Metrics {
+    pub async fn ingestor_prometheus_samples(
+        samples: Vec<PromSample>,
+        ingestor_metadata: &IngestorMetadata,
+    ) -> Result<Self, PostError> {
+        let mut metrics = Metrics::new(ingestor_metadata.domain_name.to_string());
+
+        Self::build_metrics_from_samples(samples, &mut metrics)?;
+
+        // Get additional metadata
+        let (commit_id, staging) = Self::from_about_api_response(ingestor_metadata.clone())
+            .await
+            .map_err(|err| {
+                error!("Fatal: failed to get ingestor info: {:?}", err);
+                PostError::Invalid(err.into())
+            })?;
+
+        metrics.commit = commit_id;
+        metrics.staging = staging;
+
+        Ok(metrics)
+    }
+
+    pub async fn querier_prometheus_metrics() -> Self {
+        let mut metrics = Metrics::new(get_url().to_string());
+
+        let system_metrics = get_system_metrics().expect("Failed to get system metrics");
+
+        metrics.parseable_memory_usage.total = system_metrics.memory.total;
+        metrics.parseable_memory_usage.used = system_metrics.memory.used;
+        metrics.parseable_memory_usage.total_swap = system_metrics.memory.total_swap;
+        metrics.parseable_memory_usage.used_swap = system_metrics.memory.used_swap;
+        for cpu_usage in system_metrics.cpu {
+            metrics
+                .parseable_cpu_usage
+                .insert(cpu_usage.name.clone(), cpu_usage.usage);
+        }
+
+        let staging_disk_usage = get_volume_disk_usage(CONFIG.staging_dir())
+            .expect("Failed to get staging volume disk usage");
+
+        metrics.parseable_staging_disk_usage.total = staging_disk_usage.total;
+        metrics.parseable_staging_disk_usage.used = staging_disk_usage.used;
+        metrics.parseable_staging_disk_usage.available = staging_disk_usage.available;
+
+        if CONFIG.get_storage_mode_string() == "Local drive" {
+            let data_disk_usage =
+                get_volume_disk_usage(Path::new(&CONFIG.storage().get_endpoint()))
+                    .expect("Failed to get data volume disk usage");
+
+            metrics.parseable_data_disk_usage.total = data_disk_usage.total;
+            metrics.parseable_data_disk_usage.used = data_disk_usage.used;
+            metrics.parseable_data_disk_usage.available = data_disk_usage.available;
+        }
+
+        if CONFIG.options.hot_tier_storage_path.is_some() {
+            let hot_tier_disk_usage =
+                get_volume_disk_usage(CONFIG.hot_tier_dir().as_ref().unwrap())
+                    .expect("Failed to get hot tier volume disk usage");
+
+            metrics.parseable_hot_tier_disk_usage.total = hot_tier_disk_usage.total;
+            metrics.parseable_hot_tier_disk_usage.used = hot_tier_disk_usage.used;
+            metrics.parseable_hot_tier_disk_usage.available = hot_tier_disk_usage.available;
+        }
+
+        metrics.commit = current().commit_hash;
+        metrics.staging = CONFIG.staging_dir().display().to_string();
+
+        metrics
+    }
+
+    fn build_metrics_from_samples(
+        samples: Vec<PromSample>,
+        metrics: &mut Metrics,
+    ) -> Result<(), PostError> {
+        for sample in samples {
+            let metric_type = MetricType::from_metric(&sample.metric, &sample.labels);
+
+            match (sample.value.clone(), metric_type) {
+                (PromValue::Gauge(val), Some(metric_type)) => {
+                    Self::process_gauge_metric(
+                        metrics,
+                        metric_type,
+                        val,
+                        &sample.metric,
+                        sample.clone(),
+                    );
+                }
+                _ => continue,
+            }
+        }
+        Ok(())
+    }
+
+    fn process_gauge_metric(
+        metrics: &mut Metrics,
+        metric_type: MetricType,
+        val: f64,
+        metric_name: &str,
+        sample: PromSample,
+    ) {
+        match metric_type {
+            MetricType::SimpleGauge(metric_name) => match metric_name.as_str() {
+                "parseable_events_ingested" => metrics.parseable_events_ingested += val,
+                "parseable_events_ingested_size" => metrics.parseable_events_ingested_size += val,
+                "parseable_lifetime_events_ingested" => {
+                    metrics.parseable_lifetime_events_ingested += val
+                }
+                "parseable_lifetime_events_ingested_size" => {
+                    metrics.parseable_lifetime_events_ingested_size += val
+                }
+                "parseable_events_deleted" => metrics.parseable_deleted_events_ingested += val,
+                "parseable_events_deleted_size" => {
+                    metrics.parseable_deleted_events_ingested_size += val
+                }
+                "parseable_staging_files" => metrics.parseable_staging_files += val,
+                "process_resident_memory_bytes" => metrics.process_resident_memory_bytes += val,
+                _ => {}
+            },
+            MetricType::StorageSize(storage_type) => match storage_type.as_str() {
+                "staging" => metrics.parseable_storage_size.staging += val,
+                "data" => metrics.parseable_storage_size.data += val,
+                _ => {}
+            },
+            MetricType::DiskUsage(volume_type) => {
+                let disk_usage = match volume_type.as_str() {
+                    "data" => &mut metrics.parseable_data_disk_usage,
+                    "staging" => &mut metrics.parseable_staging_disk_usage,
+                    "hot_tier" => &mut metrics.parseable_hot_tier_disk_usage,
+                    _ => return,
+                };
+
+                match metric_name {
+                    "parseable_total_disk" => disk_usage.total = val as u64,
+                    "parseable_used_disk" => disk_usage.used = val as u64,
+                    "parseable_available_disk" => disk_usage.available = val as u64,
+                    _ => {}
+                }
+            }
+            MetricType::MemoryUsage(memory_type) => match memory_type.as_str() {
+                "total_memory" => metrics.parseable_memory_usage.total = val as u64,
+                "used_memory" => metrics.parseable_memory_usage.used = val as u64,
+                "total_swap" => metrics.parseable_memory_usage.total_swap = val as u64,
+                "used_swap" => metrics.parseable_memory_usage.used_swap = val as u64,
+                _ => {}
+            },
+            MetricType::CpuUsage => {
+                if let Some(cpu_name) = sample.labels.get("cpu_usage") {
+                    metrics
+                        .parseable_cpu_usage
+                        .insert(cpu_name.to_string(), val);
+                }
+            }
+        }
+    }
+
     pub fn get_daily_stats_from_samples(
         samples: Vec<PromSample>,
         stream_name: &str,
@@ -154,68 +368,6 @@ impl Metrics {
         }
         (events_ingested, ingestion_size, storage_size)
     }
-    pub async fn from_prometheus_samples(
-        samples: Vec<PromSample>,
-        ingestor_metadata: &IngestorMetadata,
-    ) -> Result<Self, PostError> {
-        let mut prom_dress = Metrics::new(ingestor_metadata.domain_name.to_string());
-        for sample in samples {
-            if let PromValue::Gauge(val) = sample.value {
-                match sample.metric.as_str() {
-                    "parseable_events_ingested" => prom_dress.parseable_events_ingested += val,
-                    "parseable_events_ingested_size" => {
-                        prom_dress.parseable_events_ingested_size += val
-                    }
-                    "parseable_lifetime_events_ingested" => {
-                        prom_dress.parseable_lifetime_events_ingested += val
-                    }
-                    "parseable_lifetime_events_ingested_size" => {
-                        prom_dress.parseable_lifetime_events_ingested_size += val
-                    }
-                    "parseable_events_deleted" => {
-                        prom_dress.parseable_deleted_events_ingested += val
-                    }
-                    "parseable_events_deleted_size" => {
-                        prom_dress.parseable_deleted_events_ingested_size += val
-                    }
-                    "parseable_staging_files" => prom_dress.parseable_staging_files += val,
-                    "process_resident_memory_bytes" => {
-                        prom_dress.process_resident_memory_bytes += val
-                    }
-                    "parseable_storage_size" => {
-                        if sample.labels.get("type").expect("type is present") == "staging" {
-                            prom_dress.parseable_storage_size.staging += val;
-                        }
-                        if sample.labels.get("type").expect("type is present") == "data" {
-                            prom_dress.parseable_storage_size.data += val;
-                        }
-                    }
-                    "parseable_lifetime_events_storage_size" => {
-                        if sample.labels.get("type").expect("type is present") == "data" {
-                            prom_dress.parseable_lifetime_storage_size.data += val;
-                        }
-                    }
-                    "parseable_deleted_events_storage_size" => {
-                        if sample.labels.get("type").expect("type is present") == "data" {
-                            prom_dress.parseable_deleted_storage_size.data += val;
-                        }
-                    }
-                    _ => {}
-                }
-            }
-        }
-        let (commit_id, staging) = Self::from_about_api_response(ingestor_metadata.clone())
-            .await
-            .map_err(|err| {
-                error!("Fatal: failed to get ingestor info: {:?}", err);
-                PostError::Invalid(err.into())
-            })?;
-
-        prom_dress.commit = commit_id;
-        prom_dress.staging = staging;
-
-        Ok(prom_dress)
-    }
 
     pub async fn from_about_api_response(
         ingestor_metadata: IngestorMetadata,