Skip to content

Commit

Permalink
Unblock openQA jobs only on network-related Git update failures
Browse files Browse the repository at this point in the history
The starting point for the previous commit "Avoid blocking openQA jobs on
Git updates" were network-related issues, see
https://progress.opensuse.org/issues/169204. So it makes sense to limit it
to network-related changes.
  • Loading branch information
Martchus committed Nov 12, 2024
1 parent 4634d6c commit e2fc65d
Show file tree
Hide file tree
Showing 2 changed files with 27 additions and 20 deletions.
3 changes: 2 additions & 1 deletion lib/OpenQA/Task/Git/Clone.pm
Original file line number Diff line number Diff line change
Expand Up @@ -51,14 +51,15 @@ sub _git_clone_all ($job, $clones) {
eval { _git_clone($app, $job, $ctx, $path, $url) };
next unless my $error = $@;

# unblock openQA jobs despite the error under best-effort configuration
# unblock openQA jobs despite network errors under best-effort configuration
my $retries = $job->retries;
my $git_config = $app->config->{'scm git'};
my $max_retries = $ENV{OPENQA_GIT_CLONE_RETRIES} // 10;
my $max_best_effort_retries = min($max_retries, $ENV{OPENQA_GIT_CLONE_RETRIES_BEST_EFFORT} // 2);
my $gru_task_id = $job->info->{notes}->{gru_id};
if ( $is_path_only
&& defined($gru_task_id)
&& ($error =~ m/disconnect|curl|stream.*closed|/i)
&& $git_config->{git_auto_update_method} eq 'best-effort'
&& $retries >= $max_best_effort_retries)
{
Expand Down
44 changes: 25 additions & 19 deletions t/14-grutasks-git.t
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ use OpenQA::Task::Git::Clone;
require OpenQA::Test::Database;
use OpenQA::Test::Utils qw(run_gru_job perform_minion_jobs);
use OpenQA::Test::TimeLimit '20';
use Test::Output qw(stderr_like);
use Test::Output qw(combined_like stderr_like);
use Test::MockModule;
use Test::Mojo;
use Test::Warnings qw(:report_warnings);
Expand Down Expand Up @@ -260,6 +260,9 @@ subtest 'git clone' => sub {
};

subtest 'git_update_all' => sub {
my $clone_mock = Test::MockModule->new('OpenQA::Task::Git::Clone');
$clone_mock->redefine(_git_clone => sub (@args) { die 'fake disconnect' });

$t->app->config->{'scm git'}->{git_auto_update} = 'yes';
my $testdir = $workdir->child('openqa/share/tests');
$testdir->make_path;
Expand All @@ -278,24 +281,27 @@ subtest 'git_update_all' => sub {
my $gru_id = $result->{gru_id};
my $gru_task = $gru_tasks->find($gru_id);
is_deeply [sort keys %$args], \@clones, 'job args as expected';
isnt $gru_task, undef, 'gru task created' or return;

# assume an openQA job is blocked on the Gru task
my $blocked_job = $jobs->create({TEST => 'blocked-job'});
$gru_task->jobs->create({job_id => $blocked_job->id});

# perform job, it'll fail because $testdir is no actual Git repo
$minion->foreground($job->id); # already counts as retry
is $job->info->{state}, 'inactive', 'job failed but set to inactive to be retried';
is $gru_task->jobs->count, 1, 'openQA job not unblocked after first try';

# retry the job now
$minion->foreground($job->id);
is $job->info->{state}, 'inactive', 'job failed but again set to inactive to be retried';
is $gru_task->jobs->count, 0, 'openQA job unblocked after second try';

$minion->foreground($job->id);
is $job->info->{state}, 'failed', 'job failed for real after retries exhausted';
isnt $gru_task, undef, 'gru task created';

subtest 'error handling and retry behavior' => sub {
# assume an openQA job is blocked on the Gru task
my $blocked_job = $jobs->create({TEST => 'blocked-job'});
$gru_task->jobs->create({job_id => $blocked_job->id});

# perform job, it'll fail via the mocked _git_clone function
$minion->foreground($job->id); # already counts as retry
is $job->info->{state}, 'inactive', 'job failed but set to inactive to be retried';
is $gru_task->jobs->count, 1, 'openQA job not unblocked after first try';

# retry the job now
$minion->foreground($job->id);
is $job->info->{state}, 'inactive', 'job failed but again set to inactive to be retried';
is $gru_task->jobs->count, 0, 'openQA job unblocked after second try';

combined_like { $minion->foreground($job->id) } qr/fake disconnect/, 'error logged';
is $job->info->{state}, 'failed', 'job failed for real after retries exhausted';
}
if $gru_task;
};

subtest 'enqueue_git_clones' => sub {
Expand Down

0 comments on commit e2fc65d

Please sign in to comment.