Skip to content

Commit 1b56e3e

Browse files
tonyredondotonyredondo
authored
Smart iteration count based on confidence intervals and error (#64)
1 parent f6833e1 commit 1b56e3e

File tree

8 files changed

+229
-25
lines changed

8 files changed

+229
-25
lines changed

‎src/Directory.Build.props

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
<Project>
22
<PropertyGroup>
3-
<Version>0.2.1</Version>
3+
<Version>0.3.0</Version>
44
<Authors>Tony Redondo, Grégory Léocadie</Authors>
55
<TargetFrameworks>net6.0;net7.0;net8.0;net9.0</TargetFrameworks>
66
<ImplicitUsings>enable</ImplicitUsings>

‎src/TimeItSharp.Common/Configuration/Builder/ConfigBuilder.cs

+55
Original file line numberDiff line numberDiff line change
@@ -202,6 +202,61 @@ public ConfigBuilder WithDebugMode()
202202
_configuration.DebugMode = true;
203203
return this;
204204
}
205+
206+
/// <summary>
207+
/// Sets the acceptable relative width for the confidence interval where timeit will consider the results as valid and stop iterating
208+
/// </summary>
209+
/// <param name="acceptableRelativeWidth">Acceptable relative width</param>
210+
/// <returns>Configuration builder instance</returns>
211+
public ConfigBuilder WithAcceptableRelativeWidth(double acceptableRelativeWidth)
212+
{
213+
_configuration.AcceptableRelativeWidth = acceptableRelativeWidth;
214+
return this;
215+
}
216+
217+
/// <summary>
218+
/// Sets the confidence level for the confidence interval where timeit will compare the acceptable relative width
219+
/// </summary>
220+
/// <param name="confidenceLevel">Confidence level</param>
221+
/// <returns>Configuration builder instance</returns>
222+
public ConfigBuilder WithConfidenceLevel(double confidenceLevel)
223+
{
224+
_configuration.ConfidenceLevel = confidenceLevel;
225+
return this;
226+
}
227+
228+
/// <summary>
229+
/// Sets the maximum duration in minutes for all scenarios to run
230+
/// </summary>
231+
/// <param name="maximumDurationInMinutes">Maximum number of minutes</param>
232+
/// <returns>Configuration builder instance</returns>
233+
public ConfigBuilder WithMaximumDurationInMinutes(int maximumDurationInMinutes)
234+
{
235+
_configuration.MaximumDurationInMinutes = maximumDurationInMinutes;
236+
return this;
237+
}
238+
239+
/// <summary>
240+
/// Sets the interval in which timeit will evaluate the results and decide if there's error reductions.
241+
/// </summary>
242+
/// <param name="evaluationInterval">Interval in number of iterations</param>
243+
/// <returns>Configuration builder instance</returns>
244+
public ConfigBuilder WithEvaluationInterval(int evaluationInterval)
245+
{
246+
_configuration.EvaluationInterval = evaluationInterval;
247+
return this;
248+
}
249+
250+
/// <summary>
251+
/// Sets the minimum error reduction required for timeit to consider the results as valid and stop iterating
252+
/// </summary>
253+
/// <param name="minimumErrorReduction">Minimum error reduction required</param>
254+
/// <returns>Configuration builder instance</returns>
255+
public ConfigBuilder WithMinimumErrorReduction(double minimumErrorReduction)
256+
{
257+
_configuration.MinimumErrorReduction = minimumErrorReduction;
258+
return this;
259+
}
205260

206261
#region WithExporter
207262

‎src/TimeItSharp.Common/Configuration/Config.cs

+26
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,22 @@ public class Config : ProcessData
5656
[JsonPropertyName("debugMode")]
5757
public bool DebugMode { get; set; }
5858

59+
[JsonPropertyName("acceptableRelativeWidth")]
60+
public double AcceptableRelativeWidth { get; set; }
61+
62+
[JsonPropertyName("confidenceLevel")]
63+
public double ConfidenceLevel { get; set; }
64+
65+
[JsonPropertyName("maximumDurationInMinutes")]
66+
public int MaximumDurationInMinutes { get; set; }
67+
68+
[JsonPropertyName("evaluationInterval")]
69+
public int EvaluationInterval { get; set; }
70+
71+
[JsonPropertyName("minimumErrorReduction")]
72+
public double MinimumErrorReduction { get; set; }
73+
74+
5975
public Config()
6076
{
6177
FilePath = string.Empty;
@@ -75,6 +91,11 @@ public Config()
7591
ProcessFailedDataPoints = false;
7692
ShowStdOutForFirstRun = false;
7793
DebugMode = false;
94+
AcceptableRelativeWidth = 0.01;
95+
ConfidenceLevel = 0.95;
96+
MaximumDurationInMinutes = 45;
97+
EvaluationInterval = 10;
98+
MinimumErrorReduction = 0.0005;
7899
}
79100

80101
public static Config LoadConfiguration(string filePath)
@@ -133,5 +154,10 @@ public static Config LoadConfiguration(string filePath)
133154
ProcessFailedDataPoints = ProcessFailedDataPoints,
134155
ShowStdOutForFirstRun = ShowStdOutForFirstRun,
135156
DebugMode = DebugMode,
157+
AcceptableRelativeWidth = AcceptableRelativeWidth,
158+
ConfidenceLevel = ConfidenceLevel,
159+
MaximumDurationInMinutes = MaximumDurationInMinutes,
160+
EvaluationInterval = EvaluationInterval,
161+
MinimumErrorReduction = MinimumErrorReduction,
136162
};
137163
}

‎src/TimeItSharp.Common/Exporters/ConsoleExporter.cs

+8-7
Original file line numberDiff line numberDiff line change
@@ -30,21 +30,22 @@ public void Export(TimeitResult results)
3030

3131
// ****************************************
3232
// Results table
33-
AnsiConsole.MarkupLine("[aqua bold underline]### Results:[/]");
33+
AnsiConsole.MarkupLine("[aqua bold underline]### Results (last 10):[/]");
3434
var resultsTable = new Table()
3535
.MarkdownBorder();
3636

3737
// Add columns
3838
resultsTable.AddColumns(results.Scenarios.Select(r => new TableColumn($"[dodgerblue1 bold]{r.Name}[/]").Centered()).ToArray());
3939

4040
// Add rows
41-
for (var i = 0; i < _options.Configuration.Count; i++)
41+
var minDurationCount = Math.Min(results.Scenarios.Select(r => r.Durations.Count).Min(), 10);
42+
for (var i = minDurationCount; i > 0; i--)
4243
{
4344
resultsTable.AddRow(results.Scenarios.Select(r =>
4445
{
4546
if (i < r.Durations.Count)
4647
{
47-
return Math.Round(Utils.FromNanosecondsToMilliseconds(r.Durations[i]), 3) + "ms";
48+
return Math.Round(Utils.FromNanosecondsToMilliseconds(r.Durations[^i]), 3) + "ms";
4849
}
4950

5051
return "-";
@@ -56,10 +57,10 @@ public void Export(TimeitResult results)
5657

5758
// ****************************************
5859
// Outliers table
59-
var maxOutliersCount = results.Scenarios.Select(r => r.Outliers.Count).Max();
60+
var maxOutliersCount = Math.Min(results.Scenarios.Select(r => r.Outliers.Count).Max(), 5);
6061
if (maxOutliersCount > 0)
6162
{
62-
AnsiConsole.MarkupLine("[aqua bold underline]### Outliers:[/]");
63+
AnsiConsole.MarkupLine("[aqua bold underline]### Outliers (last 5):[/]");
6364
var outliersTable = new Table()
6465
.MarkdownBorder();
6566

@@ -68,13 +69,13 @@ public void Export(TimeitResult results)
6869
.Select(r => new TableColumn($"[dodgerblue1 bold]{r.Name}[/]").Centered()).ToArray());
6970

7071
// Add rows
71-
for (var i = 0; i < maxOutliersCount; i++)
72+
for (var i = maxOutliersCount; i > 0; i--)
7273
{
7374
outliersTable.AddRow(results.Scenarios.Select(r =>
7475
{
7576
if (i < r.Outliers.Count)
7677
{
77-
return Math.Round(Utils.FromNanosecondsToMilliseconds(r.Outliers[i]), 3) + "ms";
78+
return Math.Round(Utils.FromNanosecondsToMilliseconds(r.Outliers[^i]), 3) + "ms";
7879
}
7980

8081
return "-";

‎src/TimeItSharp.Common/ScenarioProcessor.cs

+128-2
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
using CliWrap;
1010
using CliWrap.Buffered;
1111
using DatadogTestLogger.Vendors.Datadog.Trace;
12+
using MathNet.Numerics.Distributions;
1213
using MathNet.Numerics.Statistics;
1314
using Spectre.Console;
1415
using TimeItSharp.Common.Assertors;
@@ -29,6 +30,8 @@ internal sealed class ScenarioProcessor
2930

3031
private static readonly IDictionary EnvironmentVariables = Environment.GetEnvironmentVariables();
3132

33+
private double _remainingTimeInMinutes;
34+
3235
public ScenarioProcessor(
3336
Config configuration,
3437
TemplateVariables templateVariables,
@@ -41,6 +44,7 @@ public ScenarioProcessor(
4144
_assertors = assertors;
4245
_services = services;
4346
_callbacksTriggers = callbacksTriggers;
47+
_remainingTimeInMinutes = configuration.MaximumDurationInMinutes;
4448
}
4549

4650
[UnconditionalSuppressMessage("SingleFile", "IL3000:Avoid accessing Assembly file path when publishing as a single file", Justification = "Case is being handled")]
@@ -212,6 +216,7 @@ public void CleanScenario(Scenario scenario)
212216
AnsiConsole.Markup(" [gold3_1]Warming up[/]");
213217
watch.Restart();
214218
await RunScenarioAsync(_configuration.WarmUpCount, index, scenario, TimeItPhase.WarmUp, false,
219+
stopwatch: watch,
215220
cancellationToken: cancellationToken).ConfigureAwait(false);
216221
watch.Stop();
217222
if (cancellationToken.IsCancellationRequested)
@@ -225,7 +230,9 @@ await RunScenarioAsync(_configuration.WarmUpCount, index, scenario, TimeItPhase.
225230
AnsiConsole.Markup(" [green3]Run[/]");
226231
var start = DateTime.UtcNow;
227232
watch.Restart();
228-
var dataPoints = await RunScenarioAsync(_configuration.Count, index, scenario, TimeItPhase.Run, true, cancellationToken: cancellationToken).ConfigureAwait(false);
233+
var dataPoints = await RunScenarioAsync(_configuration.Count, index, scenario, TimeItPhase.Run, true,
234+
stopwatch: watch,
235+
cancellationToken: cancellationToken).ConfigureAwait(false);
229236
watch.Stop();
230237
if (cancellationToken.IsCancellationRequested)
231238
{
@@ -241,6 +248,7 @@ await RunScenarioAsync(_configuration.WarmUpCount, index, scenario, TimeItPhase.
241248
scenario.ParentService = repeat.ServiceAskingForRepeat;
242249
watch.Restart();
243250
await RunScenarioAsync(repeat.Count, index, scenario, TimeItPhase.ExtraRun, false,
251+
stopwatch: watch,
244252
cancellationToken: cancellationToken).ConfigureAwait(false);
245253
watch.Stop();
246254
if (cancellationToken.IsCancellationRequested)
@@ -416,8 +424,17 @@ await RunScenarioAsync(repeat.Count, index, scenario, TimeItPhase.ExtraRun, fals
416424
return scenarioResult;
417425
}
418426

419-
private async Task<List<DataPoint>> RunScenarioAsync(int count, int index, Scenario scenario, TimeItPhase phase, bool checkShouldContinue, CancellationToken cancellationToken)
427+
private async Task<List<DataPoint>> RunScenarioAsync(int count, int index, Scenario scenario, TimeItPhase phase, bool checkShouldContinue, Stopwatch stopwatch, CancellationToken cancellationToken)
420428
{
429+
var minIterations = count / 3;
430+
minIterations = minIterations < 10 ? 10 : minIterations;
431+
var confidenceLevel = _configuration.ConfidenceLevel;
432+
if (confidenceLevel is <= 0 or >= 1)
433+
{
434+
confidenceLevel = 0.95;
435+
}
436+
var previousRelativeWidth = double.MaxValue;
437+
421438
var dataPoints = new List<DataPoint>();
422439
AnsiConsole.Markup(" ");
423440
for (var i = 0; i < count; i++)
@@ -440,10 +457,119 @@ private async Task<List<DataPoint>> RunScenarioAsync(int count, int index, Scena
440457
{
441458
break;
442459
}
460+
461+
try
462+
{
463+
// If we are in a run phase, let's do the automatic checks
464+
if (phase == TimeItPhase.Run)
465+
{
466+
static double GetDuration(DataPoint point)
467+
{
468+
#if NET7_0_OR_GREATER
469+
return point.Duration.TotalNanoseconds;
470+
#else
471+
return Utils.FromTimeSpanToNanoseconds(point.Duration);
472+
#endif
473+
}
474+
475+
var durations = Utils.RemoveOutliers(dataPoints.Select(GetDuration), threshold: 1.5).ToList();
476+
if (durations.Count >= minIterations || stopwatch.Elapsed.TotalMinutes >= _remainingTimeInMinutes)
477+
{
478+
var mean = durations.Average();
479+
var stdev = durations.StandardDeviation();
480+
var stderr = stdev / Math.Sqrt(durations.Count);
481+
482+
// Critical t value
483+
var tCritical = StudentT.InvCDF(0, 1, durations.Count - 1, 1 - (1 - confidenceLevel) / 2);
484+
485+
// Confidence intervals
486+
var marginOfError = tCritical * stderr;
487+
var confidenceIntervalLower = mean - marginOfError;
488+
var confidenceIntervalUpper = mean + marginOfError;
489+
var relativeWidth = (confidenceIntervalUpper - confidenceIntervalLower) / mean;
490+
491+
// Check if the maximum duration is reached
492+
if (stopwatch.Elapsed.TotalMinutes >= _remainingTimeInMinutes)
493+
{
494+
AnsiConsole.WriteLine();
495+
AnsiConsole.MarkupLine(
496+
" [blueviolet]Maximum duration has been reached. Stopping iterations for this scenario.[/]");
497+
AnsiConsole.MarkupLine(" [blueviolet]N: {0}[/]", durations.Count);
498+
AnsiConsole.MarkupLine(" [blueviolet]Mean: {0}ms[/]",
499+
Math.Round(Utils.FromNanosecondsToMilliseconds(mean), 3));
500+
AnsiConsole.Markup(
501+
" [blueviolet]Confidence Interval at {0}: [[{1}ms, {2}ms]]. Relative width: {3}%[/]",
502+
confidenceLevel * 100,
503+
Math.Round(Utils.FromNanosecondsToMilliseconds(confidenceIntervalLower), 3),
504+
Math.Round(Utils.FromNanosecondsToMilliseconds(confidenceIntervalUpper), 3),
505+
Math.Round(relativeWidth * 100, 4));
506+
507+
break;
508+
}
509+
510+
// Check if the statistical criterion is met
511+
if (relativeWidth < _configuration.AcceptableRelativeWidth)
512+
{
513+
AnsiConsole.WriteLine();
514+
AnsiConsole.MarkupLine(
515+
" [blueviolet]Acceptable relative width criteria met. Stopping iterations for this scenario.[/]");
516+
AnsiConsole.MarkupLine(" [blueviolet]N: {0}[/]", durations.Count);
517+
AnsiConsole.MarkupLine(" [blueviolet]Mean: {0}ms[/]",
518+
Math.Round(Utils.FromNanosecondsToMilliseconds(mean), 3));
519+
AnsiConsole.Markup(
520+
" [blueviolet]Confidence Interval at {0}: [[{1}ms, {2}ms]]. Relative width: {3}%[/]",
521+
confidenceLevel * 100,
522+
Math.Round(Utils.FromNanosecondsToMilliseconds(confidenceIntervalLower), 3),
523+
Math.Round(Utils.FromNanosecondsToMilliseconds(confidenceIntervalUpper), 3),
524+
Math.Round(relativeWidth * 100, 4));
525+
break;
526+
}
527+
528+
// Check for each `evaluationInterval` iteration
529+
if ((durations.Count - minIterations) % _configuration.EvaluationInterval == 0)
530+
{
531+
var errorReduction = (previousRelativeWidth - relativeWidth) / previousRelativeWidth;
532+
if (errorReduction < _configuration.MinimumErrorReduction)
533+
{
534+
AnsiConsole.WriteLine();
535+
AnsiConsole.MarkupLine(
536+
" [blueviolet]The error is not decreasing significantly. Stopping iterations for this scenario.[/]");
537+
AnsiConsole.MarkupLine(" [blueviolet]N: {0}[/]", durations.Count);
538+
AnsiConsole.MarkupLine(" [blueviolet]Mean: {0}ms[/]",
539+
Math.Round(Utils.FromNanosecondsToMilliseconds(mean), 3));
540+
AnsiConsole.MarkupLine(
541+
" [blueviolet]Confidence Interval at {0}: [[{1}ms, {2}ms]]. Relative width: {3}%[/]",
542+
confidenceLevel * 100,
543+
Math.Round(Utils.FromNanosecondsToMilliseconds(confidenceIntervalLower), 3),
544+
Math.Round(Utils.FromNanosecondsToMilliseconds(confidenceIntervalUpper), 3),
545+
Math.Round(relativeWidth * 100, 4));
546+
AnsiConsole.Markup(" [blueviolet]Error reduction: {0}%. Minimal expected: {1}%[/]",
547+
Math.Round(errorReduction * 100, 4),
548+
Math.Round(_configuration.MinimumErrorReduction * 100, 4));
549+
550+
break;
551+
}
552+
553+
previousRelativeWidth = relativeWidth;
554+
}
555+
}
556+
}
557+
}
558+
catch (Exception ex)
559+
{
560+
AnsiConsole.WriteLine();
561+
AnsiConsole.MarkupLine(" [red]Error: {0}[/]", ex.Message);
562+
break;
563+
}
443564
}
444565

445566
AnsiConsole.WriteLine();
446567

568+
if (phase == TimeItPhase.Run)
569+
{
570+
_remainingTimeInMinutes -= (int)stopwatch.Elapsed.TotalMinutes;
571+
}
572+
447573
return dataPoints;
448574
}
449575

‎src/TimeItSharp.Common/TimeItEngine.cs

+5-1
Original file line numberDiff line numberDiff line change
@@ -93,7 +93,11 @@ public static async Task<int> RunAsync(Config config, TimeItOptions? options = n
9393

9494
AnsiConsole.Profile.Width = Utils.GetSafeWidth();
9595
AnsiConsole.MarkupLine("[bold aqua]Warmup count:[/] {0}", config.WarmUpCount);
96-
AnsiConsole.MarkupLine("[bold aqua]Count:[/] {0}", config.Count);
96+
AnsiConsole.MarkupLine("[bold aqua]Max count:[/] {0}", config.Count);
97+
AnsiConsole.MarkupLine("[bold aqua]Acceptable relative width:[/] {0}%", Math.Round(config.AcceptableRelativeWidth * 100, 2));
98+
AnsiConsole.MarkupLine("[bold aqua]Confidence level:[/] {0}%", Math.Round(config.ConfidenceLevel * 100, 2));
99+
AnsiConsole.MarkupLine("[bold aqua]Minimum error reduction:[/] {0}%", Math.Round(config.MinimumErrorReduction * 100, 2));
100+
AnsiConsole.MarkupLine("[bold aqua]Maximum duration:[/] {0}min", config.MaximumDurationInMinutes);
97101
AnsiConsole.MarkupLine("[bold aqua]Number of Scenarios:[/] {0}", config.Scenarios.Count);
98102
AnsiConsole.MarkupLine("[bold aqua]Exporters:[/] {0}", string.Join(", ", exporters.Select(e => e.Name)));
99103
AnsiConsole.MarkupLine("[bold aqua]Assertors:[/] {0}", string.Join(", ", assertors.Select(e => e.Name)));

0 commit comments

Comments
 (0)