Add UDTF that detects linux kernel header installation and add column to GetAgentStatus (#2052)

Summary: Add UDTF that detects linux kernel header installation and add
column to `GetAgentStatus`

This is a prerequisite to accomplish #2051. The `px deploy` command uses
the GetAgentStatus UDTF in its final [healthcheck
step](https://github.com/pixie-io/pixie/blob/854062111cf4b91a40649a2e2647c88c0a68b0db/src/pixie_cli/pkg/cmd/deploy.go#L607-L613).
With this kernel header detection in place, the `px` cli can use the
results from the `px/agent_status` script to print a warning message if
kernel headers aren't detected.

The helm install flow needs to be covered as well. My hope is that this
UDTF could be used for that use case as well, but I need to further
investigate the details of that.

Relevant Issues: #2051

Type of change: /kind feature

Test Plan: Skaffolded to a Ubuntu GKE cluster and tested the following
- [x] Kelvin always reports `false` as it doesn't bind mount `/` to
`/host`
- [x] PEM running on host without `linux-headers-$(uname -r)` package
reports `false`
- [x] PEM running on host with `linux-headers-$(uname -r)` package
reports `true`
```
$ gcloud compute ssh gke-dev-cluster-ddelnano-default-pool-a27c1ac2-x5k2 --internal-ip -- 'ls -alh /lib/modules/$(uname -r)/build'

lrwxrwxrwx 1 root root 38 Aug  9 15:25 /lib/modules/5.15.0-1065-gke/build -> /usr/src/linux-headers-5.15.0-1065-gke

$ gcloud compute ssh gke-dev-cluster-ddelnano-default-pool-a27c1ac2-j6pg --internal-ip -- 'ls -alh /lib/modules/$(uname -r)/build'

ls: cannot access '/lib/modules/5.15.0-1065-gke/build': No such file or directory

```
![Screen Shot 2024-12-02 at 9 30 29
AM](https://github.com/user-attachments/assets/9fa862f8-5a6c-46d6-8899-bfaf2bdf3371)


Changelog Message: Add `GetLinuxHeadersStatus` UDTF and add
`kernel_headers_installed` column to `GetAgentStatus`

---------

Signed-off-by: Dom Del Nano <[email protected]>

Author: ddelnano

src/common/system/kernel_version.h

@@ -75,6 +75,11 @@ enum class KernelVersionOrder {
   kNewer,
 };
 
+struct KernelInfo {
+  KernelVersion version;
+  bool kernel_headers_installed;
+};
+
 // Compares two kernel versions and detect their relationship.
 KernelVersionOrder CompareKernelVersions(KernelVersion a, KernelVersion b);
 

src/common/system/linux_headers_utils.cc

@@ -0,0 +1,78 @@
+/*
+ * Copyright 2018- The Pixie Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+#include "src/common/system/linux_headers_utils.h"
+
+#include <fstream>
+#include <limits>
+#include <memory>
+#include <string>
+
+#include "src/common/base/file.h"
+#include "src/common/fs/fs_wrapper.h"
+#include "src/common/system/config.h"
+
+namespace px {
+namespace system {
+
+StatusOr<std::filesystem::path> ResolvePossibleSymlinkToHostPath(const std::filesystem::path p) {
+  // Check if "p" is a symlink.
+  std::error_code ec;
+  const bool is_symlink = std::filesystem::is_symlink(p, ec);
+  if (ec) {
+    return error::NotFound(absl::Substitute("Did not find the host headers at path: $0, $1.",
+                                            p.string(), ec.message()));
+  }
+
+  if (!is_symlink) {
+    // Not a symlink, we are good now.
+    return p;
+  }
+
+  // Resolve the symlink, and re-convert to a host path..
+  const std::filesystem::path resolved = std::filesystem::read_symlink(p, ec);
+  if (ec) {
+    return error::Internal(ec.message());
+  }
+
+  // Relative paths containing "../" can result in an invalid host mount path when using
+  // ToHostPath. Therefore, we need to treat the absolute and relative cases differently.
+  std::filesystem::path resolved_host_path;
+  if (resolved.is_absolute()) {
+    resolved_host_path = system::Config::GetInstance().ToHostPath(resolved);
+    VLOG(1) << absl::Substitute(
+        "Symlink target is an absolute path. Converting that to host path: $0 -> $1.",
+        resolved.string(), resolved_host_path.string());
+  } else {
+    resolved_host_path = p.parent_path();
+    resolved_host_path /= resolved.string();
+    VLOG(1) << absl::Substitute(
+        "Symlink target is a relative path. Concatenating it to parent directory: $0",
+        resolved_host_path.string());
+  }
+
+  // Downstream won't be ok unless the resolved host path exists; return an error if needed.
+  if (!fs::Exists(resolved_host_path)) {
+    return error::NotFound(absl::Substitute("Did not find host headers at resolved path: $0.",
+                                            resolved_host_path.string()));
+  }
+  return resolved_host_path;
+}
+
+}  // namespace system
+}  // namespace px

src/common/system/linux_headers_utils.h

@@ -0,0 +1,46 @@
+/*
+ * Copyright 2018- The Pixie Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+#pragma once
+
+#include <filesystem>
+
+#include "src/common/base/base.h"
+
+namespace px {
+namespace system {
+
+constexpr std::string_view kLinuxModulesDir = "/lib/modules/";
+
+/**
+ * Resolves a possible symlink path to its corresponding host filesystem path.
+ *
+ * This function takes a filesystem path and checks if it is a symbolic link. If it is,
+ * the symlink is resolved to its target path. Depending on whether the target is an absolute
+ * or relative path, it is further processed to convert it into a valid host path (as in
+ * Config::ToHostPath(...) path).
+ *
+ * If the input path is not a symlink, it is returned as-is. The function ensures that
+ * the final resolved path exists in the host filesystem before returning it. Errors are
+ * returned when the path does not exist, the resolution fails, or when there is an issue
+ * accessing the filesystem.
+ */
+StatusOr<std::filesystem::path> ResolvePossibleSymlinkToHostPath(const std::filesystem::path p);
+
+}  // namespace system
+}  // namespace px

src/stirling/utils/linux_headers.cc

@@ -31,6 +31,7 @@
 #include "src/common/fs/temp_file.h"
 #include "src/common/minitar/minitar.h"
 #include "src/common/system/config.h"
+#include "src/common/system/linux_headers_utils.h"
 #include "src/common/system/proc_pid_path.h"
 #include "src/common/zlib/zlib_wrapper.h"
 
@@ -83,7 +84,7 @@ StatusOr<std::filesystem::path> FindKernelConfig() {
       // Search for /boot/config-<uname>
       syscfg.ToHostPath(absl::StrCat("/boot/config-", uname)),
       // Search for /lib/modules/<uname>/config
-      syscfg.ToHostPath(absl::StrCat("/lib/modules/", uname, "/config")),
+      syscfg.ToHostPath(absl::StrCat(px::system::kLinuxModulesDir, uname, "/config")),
       // TODO(yzhao): https://github.com/lima-vm/alpine-lima/issues/67 once this issue is resolved,
       // we might consider change these 2 paths into something recommended by rancher-desktop.
       // The path used by `alpine-lima` in "Live CD" boot mechanism.
@@ -209,55 +210,12 @@ Status FindLinuxHeadersDirectory(const std::filesystem::path& lib_modules_dir) {
   return error::NotFound("Could not find 'source' or 'build' under $0.", lib_modules_dir.string());
 }
 
-StatusOr<std::filesystem::path> ResolvePossibleSymlinkToHostPath(const std::filesystem::path p) {
-  // Check if "p" is a symlink.
-  std::error_code ec;
-  const bool is_symlink = std::filesystem::is_symlink(p, ec);
-  if (ec) {
-    return error::NotFound(absl::Substitute("Did not find the host headers at path: $0, $1.",
-                                            p.string(), ec.message()));
-  }
-
-  if (!is_symlink) {
-    // Not a symlink, we are good now.
-    return p;
-  }
-
-  // Resolve the symlink, and re-convert to a host path..
-  const std::filesystem::path resolved = std::filesystem::read_symlink(p, ec);
-  if (ec) {
-    return error::Internal(ec.message());
-  }
-
-  // Relative paths containing "../" can result in an invalid host mount path when using
-  // ToHostPath. Therefore, we need to treat the absolute and relative cases differently.
-  std::filesystem::path resolved_host_path;
-  if (resolved.is_absolute()) {
-    resolved_host_path = system::Config::GetInstance().ToHostPath(resolved);
-    VLOG(1) << absl::Substitute(
-        "Symlink target is an absolute path. Converting that to host path: $0 -> $1.",
-        resolved.string(), resolved_host_path.string());
-  } else {
-    resolved_host_path = p.parent_path();
-    resolved_host_path /= resolved.string();
-    VLOG(1) << absl::Substitute(
-        "Symlink target is a relative path. Concatenating it to parent directory: $0",
-        resolved_host_path.string());
-  }
-
-  // Downstream won't be ok unless the resolved host path exists; return an error if needed.
-  if (!fs::Exists(resolved_host_path)) {
-    return error::NotFound(absl::Substitute("Did not find host headers at resolved path: $0.",
-                                            resolved_host_path.string()));
-  }
-  return resolved_host_path;
-}
-
 Status LinkHostLinuxHeadersKernel(const std::filesystem::path& lib_modules_dir) {
   const auto host_path = system::Config::GetInstance().ToHostPath(lib_modules_dir);
   LOG(INFO) << absl::Substitute("Looking for host Linux headers at $0.", host_path.string());
 
-  PX_ASSIGN_OR_RETURN(const auto resolved_host_path, ResolvePossibleSymlinkToHostPath(host_path));
+  PX_ASSIGN_OR_RETURN(const auto resolved_host_path,
+                      system::ResolvePossibleSymlinkToHostPath(host_path));
   PX_RETURN_IF_ERROR(fs::CreateSymlinkIfNotExists(resolved_host_path, lib_modules_dir));
   LOG(INFO) << absl::Substitute("Linked host headers at $0 to symlink in pem namespace at $1.",
                                 resolved_host_path.string(), lib_modules_dir.string());
@@ -401,7 +359,8 @@ Status FindOrInstallLinuxHeaders() {
   // However we find Linux headers (below) we link them into the mount namespace of this
   // process using one (or both) of the above paths.
 
-  const std::filesystem::path pem_ns_lib_modules_dir = "/lib/modules/" + uname;
+  const std::filesystem::path pem_ns_lib_modules_dir =
+      std::string(px::system::kLinuxModulesDir) + uname;
 
   // Create (or verify existence); does nothing if the directory already exists.
   PX_RETURN_IF_ERROR(fs::CreateDirectories(pem_ns_lib_modules_dir));

src/vizier/funcs/md_udtfs/md_udtfs.cc

@@ -37,6 +37,8 @@ void RegisterFuncsOrDie(const VizierFuncFactoryContext& ctx, carnot::udf::Regist
   registry->RegisterFactoryOrDie<GetProfilerSamplingPeriodMS,
                                  UDTFWithMDFactory<GetProfilerSamplingPeriodMS>>(
       "GetProfilerSamplingPeriodMS", ctx);
+  registry->RegisterFactoryOrDie<GetLinuxHeadersStatus, UDTFWithMDFactory<GetLinuxHeadersStatus>>(
+      "GetLinuxHeadersStatus", ctx);
 
   registry->RegisterOrDie<GetDebugMDState>("_DebugMDState");
   registry->RegisterFactoryOrDie<GetDebugMDWithPrefix, UDTFWithMDFactory<GetDebugMDWithPrefix>>(

src/vizier/funcs/md_udtfs/md_udtfs_impl.h

@@ -43,6 +43,10 @@ namespace px {
 namespace vizier {
 namespace funcs {
 namespace md {
+
+constexpr std::string_view kKernelHeadersInstalledDesc =
+    "Whether the agent had linux headers pre-installed";
+
 template <typename TUDTF>
 class UDTFWithMDFactory : public carnot::udf::UDTFFactory {
  public:
@@ -295,7 +299,9 @@ class GetAgentStatus final : public carnot::udf::UDTF<GetAgentStatus> {
         ColInfo("create_time", types::DataType::TIME64NS, types::PatternType::GENERAL,
                 "The creation time of the agent"),
         ColInfo("last_heartbeat_ns", types::DataType::INT64, types::PatternType::GENERAL,
-                "Time (in nanoseconds) since the last heartbeat"));
+                "Time (in nanoseconds) since the last heartbeat"),
+        ColInfo("kernel_headers_installed", types::DataType::BOOLEAN, types::PatternType::GENERAL,
+                kKernelHeadersInstalledDesc));
   }
 
   Status Init(FunctionContext*) {
@@ -330,6 +336,8 @@ class GetAgentStatus final : public carnot::udf::UDTF<GetAgentStatus> {
     rw->Append<IndexOf("agent_state")>(StringValue(magic_enum::enum_name(agent_status.state())));
     rw->Append<IndexOf("create_time")>(agent_info.create_time_ns());
     rw->Append<IndexOf("last_heartbeat_ns")>(agent_status.ns_since_last_heartbeat());
+    rw->Append<IndexOf("kernel_headers_installed")>(
+        agent_info.info().host_info().kernel_headers_installed());
 
     ++idx_;
     return idx_ < resp_->info_size();
@@ -396,6 +404,60 @@ class GetProfilerSamplingPeriodMS final : public carnot::udf::UDTF<GetProfilerSa
   std::function<void(grpc::ClientContext*)> add_context_authentication_func_;
 };
 
+/**
+ * This UDTF retrieves the status of the agents' Linux headers installation.
+ */
+class GetLinuxHeadersStatus final : public carnot::udf::UDTF<GetLinuxHeadersStatus> {
+ public:
+  using MDSStub = vizier::services::metadata::MetadataService::Stub;
+  using SchemaResponse = vizier::services::metadata::SchemaResponse;
+  GetLinuxHeadersStatus() = delete;
+  GetLinuxHeadersStatus(std::shared_ptr<MDSStub> stub,
+                        std::function<void(grpc::ClientContext*)> add_context_authentication)
+      : idx_(0), stub_(stub), add_context_authentication_func_(add_context_authentication) {}
+
+  static constexpr auto Executor() { return carnot::udfspb::UDTFSourceExecutor::UDTF_ONE_KELVIN; }
+
+  static constexpr auto OutputRelation() {
+    return MakeArray(
+        ColInfo("asid", types::DataType::INT64, types::PatternType::GENERAL, "The Agent Short ID"),
+        ColInfo("kernel_headers_installed", types::DataType::BOOLEAN, types::PatternType::GENERAL,
+                kKernelHeadersInstalledDesc));
+  }
+
+  Status Init(FunctionContext*) {
+    px::vizier::services::metadata::AgentInfoRequest req;
+    resp_ = std::make_unique<px::vizier::services::metadata::AgentInfoResponse>();
+
+    grpc::ClientContext ctx;
+    add_context_authentication_func_(&ctx);
+    auto s = stub_->GetAgentInfo(&ctx, req, resp_.get());
+    if (!s.ok()) {
+      return error::Internal("Failed to make RPC call to GetAgentInfo");
+    }
+    return Status::OK();
+  }
+
+  bool NextRecord(FunctionContext*, RecordWriter* rw) {
+    const auto& agent_metadata = resp_->info(idx_);
+    const auto& agent_info = agent_metadata.agent();
+
+    const auto asid = agent_info.asid();
+    const auto kernel_headers_installed = agent_info.info().host_info().kernel_headers_installed();
+    rw->Append<IndexOf("asid")>(asid);
+    rw->Append<IndexOf("kernel_headers_installed")>(kernel_headers_installed);
+
+    ++idx_;
+    return idx_ < resp_->info_size();
+  }
+
+ private:
+  int idx_ = 0;
+  std::unique_ptr<px::vizier::services::metadata::AgentInfoResponse> resp_;
+  std::shared_ptr<MDSStub> stub_;
+  std::function<void(grpc::ClientContext*)> add_context_authentication_func_;
+};
+
 namespace internal {
 inline rapidjson::GenericStringRef<char> StringRef(std::string_view s) {
   return rapidjson::GenericStringRef<char>(s.data(), s.size());

src/vizier/services/agent/kelvin/kelvin_main.cc

@@ -91,8 +91,12 @@ int main(int argc, char** argv) {
   LOG(INFO) << absl::Substitute("Pixie Kelvin. Version: $0, id: $1, kernel: $2",
                                 px::VersionInfo::VersionString(), agent_id.str(),
                                 kernel_version.ToString());
+  auto kernel_info = px::system::KernelInfo{
+      kernel_version,
+      false /* kernel_headers_installed */,
+  };
   auto manager = KelvinManager::Create(agent_id, FLAGS_pod_name, FLAGS_host_ip, addr,
-                                       FLAGS_rpc_port, FLAGS_nats_url, mds_addr, kernel_version)
+                                       FLAGS_rpc_port, FLAGS_nats_url, mds_addr, kernel_info)
                      .ConsumeValueOrDie();
 
   TerminationHandler::set_manager(manager.get());

src/vizier/services/agent/kelvin/kelvin_manager.h

@@ -44,9 +44,9 @@ class KelvinManager : public Manager {
   KelvinManager() = delete;
   KelvinManager(sole::uuid agent_id, std::string_view pod_name, std::string_view host_ip,
                 std::string_view addr, int grpc_server_port, std::string_view nats_url,
-                std::string_view mds_url, system::KernelVersion kernel_version)
+                std::string_view mds_url, system::KernelInfo kernel_info)
       : Manager(agent_id, pod_name, host_ip, grpc_server_port, KelvinManager::Capabilities(),
-                KelvinManager::Parameters(), nats_url, mds_url, kernel_version) {
+                KelvinManager::Parameters(), nats_url, mds_url, kernel_info) {
     info()->address = std::string(addr);
   }
 

src/vizier/services/agent/pem/pem_main.cc

@@ -25,6 +25,7 @@
 #include "src/common/base/base.h"
 #include "src/common/signal/signal.h"
 #include "src/common/system/kernel_version.h"
+#include "src/common/system/linux_headers_utils.h"
 #include "src/shared/version/version.h"
 
 DEFINE_string(nats_url, gflags::StringFromEnv("PL_NATS_URL", "pl-nats"),
@@ -68,8 +69,23 @@ int main(int argc, char** argv) {
   LOG(INFO) << absl::Substitute("Pixie PEM. Version: $0, id: $1, kernel version: $2",
                                 px::VersionInfo::VersionString(), agent_id.str(),
                                 kernel_version.ToString());
+
+  auto kernel_headers_installed = false;
+  auto uname = px::system::GetUname();
+  if (uname.ok()) {
+    const auto host_path = px::system::Config::GetInstance().ToHostPath(
+        absl::StrCat(px::system::kLinuxModulesDir, uname.ConsumeValueOrDie(), "/build"));
+
+    const auto resolved_host_path = px::system::ResolvePossibleSymlinkToHostPath(host_path);
+    kernel_headers_installed = resolved_host_path.ok();
+  }
+
+  auto kernel_info = px::system::KernelInfo{
+      kernel_version,
+      kernel_headers_installed,
+  };
   auto manager =
-      PEMManager::Create(agent_id, FLAGS_pod_name, FLAGS_host_ip, FLAGS_nats_url, kernel_version)
+      PEMManager::Create(agent_id, FLAGS_pod_name, FLAGS_host_ip, FLAGS_nats_url, kernel_info)
           .ConsumeValueOrDie();
 
   TerminationHandler::set_manager(manager.get());