Skip to content

Commit c46dac6

Browse files
authored
Ignore error when fail to report worker crash. (#3588)
1 parent e640a9f commit c46dac6

File tree

1 file changed

+26
-18
lines changed

1 file changed

+26
-18
lines changed

src/Runner.Listener/JobDispatcher.cs

+26-18
Original file line numberDiff line numberDiff line change
@@ -545,28 +545,36 @@ await processChannel.SendAsync(
545545
detailInfo = string.Join(Environment.NewLine, workerOutput);
546546
Trace.Info($"Return code {returnCode} indicate worker encounter an unhandled exception or app crash, attach worker stdout/stderr to JobRequest result.");
547547

548-
var jobServer = await InitializeJobServerAsync(systemConnection);
549-
var unhandledExceptionIssue = new Issue() { Type = IssueType.Error, Message = detailInfo };
550-
unhandledExceptionIssue.Data[Constants.Runner.InternalTelemetryIssueDataKey] = Constants.Runner.WorkerCrash;
551-
switch (jobServer)
548+
try
552549
{
553-
case IJobServer js:
554-
{
555-
await LogWorkerProcessUnhandledException(js, message, unhandledExceptionIssue);
556-
// Go ahead to finish the job with result 'Failed' if the STDERR from worker is System.IO.IOException, since it typically means we are running out of disk space.
557-
if (detailInfo.Contains(typeof(System.IO.IOException).ToString(), StringComparison.OrdinalIgnoreCase))
550+
var jobServer = await InitializeJobServerAsync(systemConnection);
551+
var unhandledExceptionIssue = new Issue() { Type = IssueType.Error, Message = detailInfo };
552+
unhandledExceptionIssue.Data[Constants.Runner.InternalTelemetryIssueDataKey] = Constants.Runner.WorkerCrash;
553+
switch (jobServer)
554+
{
555+
case IJobServer js:
558556
{
559-
Trace.Info($"Finish job with result 'Failed' due to IOException.");
560-
await ForceFailJob(js, message);
557+
await LogWorkerProcessUnhandledException(js, message, unhandledExceptionIssue);
558+
// Go ahead to finish the job with result 'Failed' if the STDERR from worker is System.IO.IOException, since it typically means we are running out of disk space.
559+
if (detailInfo.Contains(typeof(System.IO.IOException).ToString(), StringComparison.OrdinalIgnoreCase))
560+
{
561+
Trace.Info($"Finish job with result 'Failed' due to IOException.");
562+
await ForceFailJob(js, message);
563+
}
564+
565+
break;
561566
}
562-
567+
case IRunServer rs:
568+
await ForceFailJob(rs, message, unhandledExceptionIssue);
563569
break;
564-
}
565-
case IRunServer rs:
566-
await ForceFailJob(rs, message, unhandledExceptionIssue);
567-
break;
568-
default:
569-
throw new NotSupportedException($"JobServer type '{jobServer.GetType().Name}' is not supported.");
570+
default:
571+
throw new NotSupportedException($"JobServer type '{jobServer.GetType().Name}' is not supported.");
572+
}
573+
}
574+
catch (Exception ex)
575+
{
576+
Trace.Error($"Catch exception during log worker process unhandled exception.");
577+
Trace.Error(ex);
570578
}
571579
}
572580

0 commit comments

Comments
 (0)