Revert "tp: reduce shell binary size by 100KB by removing JSON export"

This reverts commit fb070d0.

Reason for revert: blocking google3 roll 

Change-Id: I22fdaca2727cf111ec1dac1ca62ecd8fc693cfbc

Android.bp

@@ -2292,6 +2292,7 @@ cc_test {
         ":perfetto_include_perfetto_ext_ipc_ipc",
         ":perfetto_include_perfetto_ext_protozero_protozero",
         ":perfetto_include_perfetto_ext_trace_processor_demangle",
+        ":perfetto_include_perfetto_ext_trace_processor_export_json",
         ":perfetto_include_perfetto_ext_trace_processor_importers_memory_tracker_memory_tracker",
         ":perfetto_include_perfetto_ext_traced_sys_stats_counters",
         ":perfetto_include_perfetto_ext_traced_traced",
@@ -2470,6 +2471,7 @@ cc_test {
         ":perfetto_src_trace_processor_db_compare",
         ":perfetto_src_trace_processor_db_db",
         ":perfetto_src_trace_processor_db_minimal",
+        ":perfetto_src_trace_processor_export_json",
         ":perfetto_src_trace_processor_importers_android_bugreport_android_bugreport",
         ":perfetto_src_trace_processor_importers_android_bugreport_android_dumpstate_event",
         ":perfetto_src_trace_processor_importers_android_bugreport_android_log_event",
@@ -15469,6 +15471,7 @@ cc_test {
         ":perfetto_include_perfetto_ext_ipc_ipc",
         ":perfetto_include_perfetto_ext_protozero_protozero",
         ":perfetto_include_perfetto_ext_trace_processor_demangle",
+        ":perfetto_include_perfetto_ext_trace_processor_export_json",
         ":perfetto_include_perfetto_ext_trace_processor_importers_memory_tracker_memory_tracker",
         ":perfetto_include_perfetto_ext_trace_processor_rpc_query_result_serializer",
         ":perfetto_include_perfetto_ext_traced_sys_stats_counters",
@@ -15689,6 +15692,7 @@ cc_test {
         ":perfetto_src_trace_processor_db_db",
         ":perfetto_src_trace_processor_db_minimal",
         ":perfetto_src_trace_processor_db_unittests",
+        ":perfetto_src_trace_processor_export_json",
         ":perfetto_src_trace_processor_importers_android_bugreport_android_bugreport",
         ":perfetto_src_trace_processor_importers_android_bugreport_android_dumpstate_event",
         ":perfetto_src_trace_processor_importers_android_bugreport_android_log_event",
@@ -16697,6 +16701,7 @@ cc_binary {
         ":perfetto_include_perfetto_ext_base_version",
         ":perfetto_include_perfetto_ext_protozero_protozero",
         ":perfetto_include_perfetto_ext_trace_processor_demangle",
+        ":perfetto_include_perfetto_ext_trace_processor_export_json",
         ":perfetto_include_perfetto_ext_trace_processor_importers_memory_tracker_memory_tracker",
         ":perfetto_include_perfetto_ext_trace_processor_rpc_query_result_serializer",
         ":perfetto_include_perfetto_ext_traced_sys_stats_counters",
@@ -16765,6 +16770,7 @@ cc_binary {
         ":perfetto_src_trace_processor_db_compare",
         ":perfetto_src_trace_processor_db_db",
         ":perfetto_src_trace_processor_db_minimal",
+        ":perfetto_src_trace_processor_export_json",
         ":perfetto_src_trace_processor_importers_android_bugreport_android_bugreport",
         ":perfetto_src_trace_processor_importers_android_bugreport_android_dumpstate_event",
         ":perfetto_src_trace_processor_importers_android_bugreport_android_log_event",

BUILD

@@ -327,6 +327,7 @@ perfetto_cc_library(
         ":src_trace_processor_db_compare",
         ":src_trace_processor_db_db",
         ":src_trace_processor_db_minimal",
+        ":src_trace_processor_export_json",
         ":src_trace_processor_importers_android_bugreport_android_bugreport",
         ":src_trace_processor_importers_android_bugreport_android_dumpstate_event",
         ":src_trace_processor_importers_android_bugreport_android_log_event",
@@ -430,6 +431,7 @@ perfetto_cc_library(
         ":include_perfetto_ext_base_base",
         ":include_perfetto_ext_protozero_protozero",
         ":include_perfetto_ext_trace_processor_demangle",
+        ":include_perfetto_ext_trace_processor_export_json",
         ":include_perfetto_ext_trace_processor_importers_memory_tracker_memory_tracker",
         ":include_perfetto_ext_trace_processor_rpc_query_result_serializer",
         ":include_perfetto_ext_traced_sys_stats_counters",
@@ -6700,6 +6702,7 @@ perfetto_cc_library(
         ":src_trace_processor_db_compare",
         ":src_trace_processor_db_db",
         ":src_trace_processor_db_minimal",
+        ":src_trace_processor_export_json",
         ":src_trace_processor_importers_android_bugreport_android_bugreport",
         ":src_trace_processor_importers_android_bugreport_android_dumpstate_event",
         ":src_trace_processor_importers_android_bugreport_android_log_event",
@@ -6801,6 +6804,7 @@ perfetto_cc_library(
         ":include_perfetto_base_base",
         ":include_perfetto_ext_base_base",
         ":include_perfetto_ext_trace_processor_demangle",
+        ":include_perfetto_ext_trace_processor_export_json",
         ":include_perfetto_ext_trace_processor_importers_memory_tracker_memory_tracker",
         ":include_perfetto_ext_traced_sys_stats_counters",
         ":include_perfetto_protozero_protozero",
@@ -6890,6 +6894,7 @@ perfetto_cc_binary(
         ":include_perfetto_ext_base_base",
         ":include_perfetto_ext_protozero_protozero",
         ":include_perfetto_ext_trace_processor_demangle",
+        ":include_perfetto_ext_trace_processor_export_json",
         ":include_perfetto_ext_trace_processor_importers_memory_tracker_memory_tracker",
         ":include_perfetto_ext_trace_processor_rpc_query_result_serializer",
         ":include_perfetto_ext_traced_sys_stats_counters",
@@ -6912,6 +6917,7 @@ perfetto_cc_binary(
         ":src_trace_processor_db_compare",
         ":src_trace_processor_db_db",
         ":src_trace_processor_db_minimal",
+        ":src_trace_processor_export_json",
         ":src_trace_processor_importers_android_bugreport_android_bugreport",
         ":src_trace_processor_importers_android_bugreport_android_dumpstate_event",
         ":src_trace_processor_importers_android_bugreport_android_log_event",

CHANGELOG

@@ -2,8 +2,7 @@ Unreleased:
   Tracing service and probes:
     *
   SQL Standard library:
-    * Removed the `EXPORT_JSON` function due to niche usecases and large binary
-      size impact; `traceconv json` is the recommended replacement.
+    *
   Trace Processor:
     *
   UI:

src/trace_processor/perfetto_sql/intrinsics/functions/BUILD.gn

@@ -60,6 +60,7 @@ source_set("functions") {
   deps = [
     ":tables",
     "../../..:demangle",
+    "../../..:export_json",
     "../../..:metatrace",
     "../../../../../gn:default_deps",
     "../../../../../gn:sqlite",

src/trace_processor/perfetto_sql/intrinsics/functions/utils.h

@@ -18,34 +18,57 @@
 #define SRC_TRACE_PROCESSOR_PERFETTO_SQL_INTRINSICS_FUNCTIONS_UTILS_H_
 
 #include <sqlite3.h>
-#include <cerrno>
-#include <cstddef>
-#include <cstdint>
-#include <cstdio>
-#include <cstdlib>
-#include <cstring>
-#include <memory>
-#include <optional>
-#include <string>
-#include <string_view>
-
-#include "perfetto/base/logging.h"
-#include "perfetto/base/status.h"
+#include <unordered_map>
+
+#include "perfetto/base/compiler.h"
 #include "perfetto/ext/base/base64.h"
 #include "perfetto/ext/base/file_utils.h"
-#include "perfetto/ext/base/hash.h"
-#include "perfetto/ext/base/scoped_file.h"
-#include "perfetto/ext/base/utils.h"
+#include "perfetto/ext/base/string_utils.h"
 #include "perfetto/ext/trace_processor/demangle.h"
-#include "perfetto/trace_processor/basic_types.h"
+#include "protos/perfetto/common/builtin_clock.pbzero.h"
+#include "src/trace_processor/db/column/utils.h"
+#include "src/trace_processor/export_json.h"
+#include "src/trace_processor/importers/common/clock_tracker.h"
 #include "src/trace_processor/perfetto_sql/intrinsics/functions/sql_function.h"
 #include "src/trace_processor/sqlite/sqlite_utils.h"
-#include "src/trace_processor/storage/trace_storage.h"
-#include "src/trace_processor/types/variadic.h"
 #include "src/trace_processor/util/regex.h"
 #include "src/trace_processor/util/status_macros.h"
 
-namespace perfetto::trace_processor {
+namespace perfetto {
+namespace trace_processor {
+
+struct ExportJson : public SqlFunction {
+  using Context = TraceStorage;
+  static base::Status Run(TraceStorage* storage,
+                          size_t /*argc*/,
+                          sqlite3_value** argv,
+                          SqlValue& /*out*/,
+                          Destructors&);
+};
+
+base::Status ExportJson::Run(TraceStorage* storage,
+                             size_t /*argc*/,
+                             sqlite3_value** argv,
+                             SqlValue& /*out*/,
+                             Destructors&) {
+  base::ScopedFstream output;
+  if (sqlite3_value_type(argv[0]) == SQLITE_INTEGER) {
+    // Assume input is an FD.
+    output.reset(fdopen(sqlite3_value_int(argv[0]), "w"));
+    if (!output) {
+      return base::ErrStatus(
+          "EXPORT_JSON: Couldn't open output file from given FD");
+    }
+  } else {
+    const char* filename =
+        reinterpret_cast<const char*>(sqlite3_value_text(argv[0]));
+    output = base::OpenFstream(filename, "w");
+    if (!output) {
+      return base::ErrStatus("EXPORT_JSON: Couldn't open output file");
+    }
+  }
+  return json::ExportJson(storage, output.get());
+}
 
 struct Hash : public SqlFunction {
   static base::Status Run(void*,
@@ -140,7 +163,7 @@ base::Status Base64Encode::Run(void*,
   if (sqlite3_value_type(value) != SQLITE_BLOB)
     return base::ErrStatus("Base64Encode only supports bytes argument");
 
-  auto byte_count = static_cast<size_t>(sqlite3_value_bytes(value));
+  size_t byte_count = static_cast<size_t>(sqlite3_value_bytes(value));
   std::string res = base::Base64Encode(sqlite3_value_blob(value), byte_count);
 
   std::unique_ptr<char, base::FreeDeleter> s(
@@ -271,7 +294,7 @@ base::Status ExtractArg::Run(TraceStorage* storage,
   if (sqlite3_value_type(argv[1]) != SQLITE_TEXT)
     return base::ErrStatus("EXTRACT_ARG: 2nd argument should be key");
 
-  auto arg_set_id = static_cast<uint32_t>(sqlite3_value_int(argv[0]));
+  uint32_t arg_set_id = static_cast<uint32_t>(sqlite3_value_int(argv[0]));
   const char* key = reinterpret_cast<const char*>(sqlite3_value_text(argv[1]));
 
   std::optional<Variadic> opt_value;
@@ -377,6 +400,7 @@ struct Regex : public SqlFunction {
   }
 };
 
-}  // namespace perfetto::trace_processor
+}  // namespace trace_processor
+}  // namespace perfetto
 
 #endif  // SRC_TRACE_PROCESSOR_PERFETTO_SQL_INTRINSICS_FUNCTIONS_UTILS_H_

src/trace_processor/trace_processor_impl.cc

@@ -900,6 +900,8 @@ void TraceProcessorImpl::InitPerfettoSqlEngine() {
   RegisterFunction<SourceGeq>(engine_.get(), "SOURCE_GEQ", -1);
   RegisterFunction<TablePtrBind>(engine_.get(), "__intrinsic_table_ptr_bind",
                                  -1);
+  RegisterFunction<ExportJson>(engine_.get(), "EXPORT_JSON", 1,
+                               context_.storage.get(), false);
   RegisterFunction<ExtractArg>(engine_.get(), "EXTRACT_ARG", 2,
                                context_.storage.get());
   RegisterFunction<AbsTimeStr>(engine_.get(), "ABS_TIME_STR", 1,

src/traceconv/BUILD.gn

@@ -87,10 +87,8 @@ source_set("lib") {
     "../../src/profiling/symbolizer",
     "../../src/profiling/symbolizer:symbolize_database",
     "../../src/protozero:proto_ring_buffer",
-    "../../src/trace_processor:export_json",
     "../../src/trace_processor:lib",
     "../../src/trace_processor:storage_minimal",
-    "../../src/trace_processor/util",
     "../../src/trace_processor/util:descriptors",
     "../../src/trace_processor/util:gzip",
     "../../src/trace_processor/util:protozero_to_text",

src/traceconv/trace_to_json.cc

@@ -17,21 +17,16 @@
 #include "src/traceconv/trace_to_json.h"
 
 #include <stdio.h>
-#include <istream>
-#include <memory>
-#include <ostream>
-#include <string>
 
 #include "perfetto/base/logging.h"
-#include "perfetto/base/status.h"
-#include "perfetto/ext/trace_processor/export_json.h"
-#include "perfetto/trace_processor/basic_types.h"
+#include "perfetto/ext/base/scoped_file.h"
+#include "perfetto/ext/base/string_utils.h"
+#include "perfetto/ext/base/temp_file.h"
 #include "perfetto/trace_processor/trace_processor.h"
-#include "src/trace_processor/util/status_macros.h"
-#include "src/traceconv/trace_to_systrace.h"
 #include "src/traceconv/utils.h"
 
-namespace perfetto::trace_to_text {
+namespace perfetto {
+namespace trace_to_text {
 
 namespace {
 
@@ -48,24 +43,35 @@ bool ExportUserspaceEvents(trace_processor::TraceProcessor* tp,
   fprintf(stderr, "Converting userspace events%c", kProgressChar);
   fflush(stderr);
 
-  struct StringWriter : public trace_processor::json::OutputWriter {
-    base::Status AppendString(const std::string& s) override {
-      res.append(s);
-      return base::OkStatus();
-    }
-    std::string res;
-  };
-
-  StringWriter string_writer;
-  base::Status status = trace_processor::json::ExportJson(tp, &string_writer);
-  if (!status.ok()) {
+  // Write userspace trace to a temporary file.
+  // TODO(eseckler): Support streaming the result out of TP directly instead.
+  auto file = base::TempFile::Create();
+  base::StackString<100> query("select export_json(\"%s\")",
+                               file.path().c_str());
+  auto it = tp->ExecuteQuery(query.ToStdString());
+
+  if (!it.Next()) {
+    auto status = it.Status();
+    PERFETTO_CHECK(!status.ok());
     PERFETTO_ELOG("Could not convert userspace events: %s", status.c_message());
     return false;
   }
 
-  // Skip writing the closing brace since we'll append system trace data.
-  writer->Write(string_writer.res.data(), string_writer.res.size() - 1);
+  base::ScopedFstream source(fopen(file.path().c_str(), "r"));
+  if (!source) {
+    PERFETTO_ELOG("Could not convert userspace events: Couldn't read file %s",
+                  file.path().c_str());
+    return false;
+  }
 
+  char buf[BUFSIZ];
+  size_t size;
+  while ((size = fread(buf, sizeof(char), BUFSIZ, *source)) > 0) {
+    // Skip writing the closing brace since we'll append system trace data.
+    if (feof(*source))
+      size--;
+    writer->Write(buf, size);
+  }
   return true;
 }
 
@@ -108,4 +114,5 @@ int TraceToJson(std::istream* input,
   return 0;
 }
 
-}  // namespace perfetto::trace_to_text
+}  // namespace trace_to_text
+}  // namespace perfetto