Add contextual request support to sourcekit-lsp diagnose

Teach `sourcekit-lsp diagnose` how to extract contextual requests from the system log and use them to reduce sourcekitd crashes.

Author: ahoppen

Sources/Diagnose/CMakeLists.txt

@@ -19,6 +19,7 @@ add_library(Diagnose STATIC
   StderrStreamConcurrencySafe.swift
   SwiftFrontendCrashScraper.swift
   Toolchain+SwiftFrontend.swift
+  Toolchain+PluginPaths.swift
   TraceFromSignpostsCommand.swift)
 
 set_target_properties(Diagnose PROPERTIES

Sources/Diagnose/DiagnoseCommand.swift

@@ -13,6 +13,7 @@
 package import ArgumentParser
 import Foundation
 import LanguageServerProtocolExtensions
+import SKLogging
 import SwiftExtensions
 import TSCExtensions
 import ToolchainRegistry
@@ -136,6 +137,7 @@ package struct DiagnoseCommand: AsyncParsableCommand {
         break
       } catch {
         // Reducing this request failed. Continue reducing the next one, maybe that one succeeds.
+        logger.info("Reducing sourcekitd crash failed: \(error.forLogging)")
       }
     }
   }
@@ -173,6 +175,7 @@ package struct DiagnoseCommand: AsyncParsableCommand {
 
       let executor = OutOfProcessSourceKitRequestExecutor(
         sourcekitd: sourcekitd,
+        pluginPaths: toolchain.pluginPaths,
         swiftFrontend: crashInfo.swiftFrontend,
         reproducerPredicate: nil
       )
@@ -457,6 +460,7 @@ package struct DiagnoseCommand: AsyncParsableCommand {
     let requestInfo = requestInfo
     let executor = OutOfProcessSourceKitRequestExecutor(
       sourcekitd: sourcekitd,
+      pluginPaths: toolchain.pluginPaths,
       swiftFrontend: swiftFrontend,
       reproducerPredicate: nil
     )

Sources/Diagnose/MergeSwiftFiles.swift

@@ -31,6 +31,7 @@ extension RequestInfo {
     let compilerArgs = compilerArgs.filter { $0 != "-primary-file" && !$0.hasSuffix(".swift") } + ["$FILE"]
     let mergedRequestInfo = RequestInfo(
       requestTemplate: requestTemplate,
+      contextualRequestTemplates: contextualRequestTemplates,
       offset: offset,
       compilerArgs: compilerArgs,
       fileContents: mergedFile

Sources/Diagnose/OSLogScraper.swift

@@ -12,6 +12,8 @@
 
 #if canImport(OSLog)
 import OSLog
+import SKLogging
+import RegexBuilder
 
 /// Reads oslog messages to find recent sourcekitd crashes.
 struct OSLogScraper {
@@ -45,34 +47,90 @@ struct OSLogScraper {
         #"subsystem CONTAINS "sourcekit-lsp" AND composedMessage CONTAINS "sourcekitd crashed" AND category = %@"#,
       logCategory
     )
-    var isInFileContentSection = false
+    enum LogSection {
+      case request
+      case fileContents
+      case contextualRequest
+    }
+    var section = LogSection.request
     var request = ""
     var fileContents = ""
+    var contextualRequests: [String] = []
+    let sourcekitdCrashedRegex = Regex {
+      "sourcekitd crashed ("
+      OneOrMore(.digit)
+      "/"
+      OneOrMore(.digit)
+      ")"
+    }
+    let contextualRequestRegex = Regex {
+      "Contextual request "
+      OneOrMore(.digit)
+      " / "
+      OneOrMore(.digit)
+      ":"
+    }
+
     for entry in try getLogEntries(matching: predicate) {
       for line in entry.composedMessage.components(separatedBy: "\n") {
-        if line.starts(with: "sourcekitd crashed (") {
+        if try sourcekitdCrashedRegex.wholeMatch(in: line) != nil {
           continue
         }
         if line == "Request:" {
           continue
         }
         if line == "File contents:" {
-          isInFileContentSection = true
+          section = .fileContents
+          continue
+        }
+        if line == "File contents:" {
+          section = .fileContents
+          continue
+        }
+        if try contextualRequestRegex.wholeMatch(in: line) != nil {
+          section = .contextualRequest
+          contextualRequests.append("")
           continue
         }
         if line == "--- End Chunk" {
           continue
         }
-        if isInFileContentSection {
-          fileContents += line + "\n"
-        } else {
+        switch section {
+        case .request:
           request += line + "\n"
+        case .fileContents:
+          fileContents += line + "\n"
+        case .contextualRequest:
+          if !contextualRequests.isEmpty {
+            contextualRequests[contextualRequests.count - 1] += line + "\n"
+          } else {
+            // Should never happen because we have appended at least one element to `contextualRequests` when switching
+            // to the `contextualRequest` section.
+            logger.fault("Dropping contextual request line: \(line)")
+          }
         }
       }
     }
 
     var requestInfo = try RequestInfo(request: request)
+
+    let contextualRequestInfos = contextualRequests.compactMap { contextualRequest in
+      orLog("Processsing contextual request") {
+        try RequestInfo(request: contextualRequest)
+      }
+    }.filter { contextualRequest in
+      if contextualRequest.fileContents != requestInfo.fileContents {
+        logger.error("Contextual request concerns a different file than the crashed request. Ignoring it")
+        return false
+      }
+      return true
+    }
+    requestInfo.contextualRequestTemplates = contextualRequestInfos.map(\.requestTemplate)
+    if requestInfo.compilerArgs.isEmpty {
+      requestInfo.compilerArgs = contextualRequestInfos.last(where: { !$0.compilerArgs.isEmpty })?.compilerArgs ?? []
+    }
     requestInfo.fileContents = fileContents
+
     return requestInfo
   }
 

Sources/Diagnose/ReduceCommand.swift

@@ -12,6 +12,7 @@
 
 package import ArgumentParser
 import Foundation
+import SourceKitD
 import ToolchainRegistry
 
 import struct TSCBasic.AbsolutePath
@@ -68,12 +69,16 @@ package struct ReduceCommand: AsyncParsableCommand {
 
   @MainActor
   package func run() async throws {
-    guard let sourcekitd = try await toolchain?.sourcekitd else {
+    guard let toolchain = try await toolchain else {
+      throw GenericError("Unable to find toolchain")
+    }
+    guard let sourcekitd = toolchain.sourcekitd else {
       throw GenericError("Unable to find sourcekitd.framework")
     }
-    guard let swiftFrontend = try await toolchain?.swiftFrontend else {
+    guard let swiftFrontend = toolchain.swiftFrontend else {
       throw GenericError("Unable to find sourcekitd.framework")
     }
+    let pluginPaths = toolchain.pluginPaths
 
     let progressBar = PercentProgressAnimation(stream: stderrStreamConcurrencySafe, header: "Reducing sourcekitd issue")
 
@@ -82,6 +87,7 @@ package struct ReduceCommand: AsyncParsableCommand {
 
     let executor = OutOfProcessSourceKitRequestExecutor(
       sourcekitd: sourcekitd,
+      pluginPaths: pluginPaths,
       swiftFrontend: swiftFrontend,
       reproducerPredicate: nsPredicate
     )
@@ -96,6 +102,6 @@ package struct ReduceCommand: AsyncParsableCommand {
     try reduceRequestInfo.fileContents.write(to: reducedSourceFile, atomically: true, encoding: .utf8)
 
     print("Reduced Request:")
-    print(try reduceRequestInfo.request(for: reducedSourceFile))
+    print(try reduceRequestInfo.requests(for: reducedSourceFile).joined(separator: "\n\n\n\n"))
   }
 }

Sources/Diagnose/ReduceFrontendCommand.swift

@@ -90,6 +90,7 @@ package struct ReduceFrontendCommand: AsyncParsableCommand {
 
     let executor = OutOfProcessSourceKitRequestExecutor(
       sourcekitd: sourcekitd,
+      pluginPaths: nil,
       swiftFrontend: swiftFrontend,
       reproducerPredicate: nsPredicate
     )

Sources/Diagnose/ReproducerBundle.swift

@@ -40,12 +40,14 @@ func makeReproducerBundle(for requestInfo: RequestInfo, toolchain: Toolchain, bu
       + requestInfo.compilerArgs.replacing(["$FILE"], with: ["./input.swift"]).joined(separator: " \\\n")
     try command.write(to: bundlePath.appendingPathComponent("command.sh"), atomically: true, encoding: .utf8)
   } else {
-    let request = try requestInfo.request(for: URL(fileURLWithPath: "/input.swift"))
-    try request.write(
-      to: bundlePath.appendingPathComponent("request.yml"),
-      atomically: true,
-      encoding: .utf8
-    )
+    let requests = try requestInfo.requests(for: bundlePath.appendingPathComponent("input.swift"))
+    for (index, request) in requests.enumerated() {
+      try request.write(
+        to: bundlePath.appendingPathComponent("request-\(index).yml"),
+        atomically: true,
+        encoding: .utf8
+      )
+    }
   }
   for compilerArg in requestInfo.compilerArgs {
     // Find the first slash so we are also able to copy files from eg.