From 5b3e208bc016958dab2b70b25bb9240de5a56efe Mon Sep 17 00:00:00 2001 From: Dylan Hardison Date: Sun, 18 Dec 2022 13:19:15 -0800 Subject: [PATCH 1/6] reformat --- Bugzilla/WebService/Bug.pm | 71 ++++++++++++++++++++------------------ 1 file changed, 38 insertions(+), 33 deletions(-) diff --git a/Bugzilla/WebService/Bug.pm b/Bugzilla/WebService/Bug.pm index 5309f8bb14..7f90082133 100644 --- a/Bugzilla/WebService/Bug.pm +++ b/Bugzilla/WebService/Bug.pm @@ -116,6 +116,7 @@ our %api_field_types = ( ); our %api_field_names = reverse %{Bugzilla::Bug::FIELD_MAP()}; + # This doesn't normally belong in FIELD_MAP, but we do want to translate # "bug_group" back into "groups". $api_field_names{'bug_group'} = 'groups'; @@ -171,7 +172,7 @@ sub fields { foreach my $field (@fields) { my $visibility_field = $field->visibility_field ? $field->visibility_field->name : undef; - my $vis_values = $field->visibility_values; + my $vis_values = $field->visibility_values; my $value_field = $field->value_field ? $field->value_field->name : undef; my (@values, $has_values); @@ -180,7 +181,7 @@ sub fields { or $field->name eq 'keywords') { $has_values = 1; - @values = @{$self->_legal_field_values({field => $field})}; + @values = @{$self->_legal_field_values({field => $field})}; } if (grep($_ eq $field->name, PRODUCT_SPECIFIC_FIELDS)) { @@ -188,19 +189,19 @@ sub fields { } my %field_data = ( - id => $self->type('int', $field->id), - type => $self->type('int', $field->type), - is_custom => $self->type('boolean', $field->custom), - name => $self->type('string', $field->name), - display_name => $self->type('string', $field->description), - is_mandatory => $self->type('boolean', $field->is_mandatory), - is_on_bug_entry => $self->type('boolean', $field->enter_bug), - visibility_field => $self->type('string', $visibility_field), + id => $self->type('int', $field->id), + type => $self->type('int', $field->type), + is_custom => $self->type('boolean', $field->custom), + name => $self->type('string', $field->name), + display_name => $self->type('string', $field->description), + is_mandatory => $self->type('boolean', $field->is_mandatory), + is_on_bug_entry => $self->type('boolean', $field->enter_bug), + visibility_field => $self->type('string', $visibility_field), visibility_values => [map { $self->type('string', $_->name) } @$vis_values], ); if ($has_values) { $field_data{value_field} = $self->type('string', $value_field); - $field_data{values} = \@values; + $field_data{values} = \@values; } push(@fields_out, filter $params, \%field_data); } @@ -318,8 +319,8 @@ sub comments { {function => 'Bug.comments', params => ['ids', 'comment_ids']}); } - my $bug_ids = $params->{ids} || []; - my $comment_ids = $params->{comment_ids} || []; + my $bug_ids = $params->{ids} || []; + my $comment_ids = $params->{comment_ids} || []; my $skip_private = $params->{skip_private} ? 1 : 0; my $dbh = Bugzilla->switch_to_shadow_db(); @@ -330,6 +331,7 @@ sub comments { } if ($skip_private) { + # Cache permissions for bugs. This highly reduces the number of calls to the DB. # visible_bugs() is only able to handle bug IDs, so we have to skip aliases. my @int = grep { $_ =~ /^\d+$/ } @$bug_ids; @@ -343,7 +345,8 @@ sub comments { if ($skip_private) { $bug = Bugzilla::Bug->new({id => $bug_id, cache => 1}); next if $bug->error || !$user->can_see_bug($bug->id); - } else { + } + else { $bug = Bugzilla::Bug->check($bug_id); } @@ -362,7 +365,7 @@ sub comments { my %comments; if (scalar @$comment_ids) { - my @ids = map { trim($_) } @$comment_ids; + my @ids = map { trim($_) } @$comment_ids; my $comment_data = Bugzilla::Comment->new_from_list(\@ids); # See if we were passed any invalid comment ids. @@ -501,6 +504,7 @@ sub history { my $skip_private = $params->{skip_private} ? 1 : 0; if ($skip_private) { + # Cache permissions for bugs. This highly reduces the number of calls to the DB. # visible_bugs() is only able to handle bug IDs, so we have to skip aliases. my @int = grep { $_ =~ /^\d+$/ } @$ids; @@ -515,7 +519,8 @@ sub history { if ($skip_private) { $bug = Bugzilla::Bug->new({id => $bug_id, cache => 1}); next if $bug->error || !$user->can_see_bug($bug->id); - } else { + } + else { $bug = Bugzilla::Bug->check($bug_id); } @@ -586,7 +591,7 @@ sub search { # Allow to search only in bug description (initial comment) if (defined $match_params->{description}) { - $match_params->{longdesc} = delete $match_params->{description}; + $match_params->{longdesc} = delete $match_params->{description}; $match_params->{longdesc_initial} = 1; } @@ -595,7 +600,7 @@ sub search { my %options = (fields => ['bug_id']); # Find the highest custom field id - my @field_ids = grep(/^f(\d+)$/, keys %$match_params); + my @field_ids = grep(/^f(\d+)$/, keys %$match_params); my $last_field_id = @field_ids ? max @field_ids + 1 : 1; # Do special search types for certain fields. @@ -837,7 +842,7 @@ sub update { my %changes = %{$all_changes{$bug->id}}; foreach my $field (keys %changes) { - my $change = $changes{$field}; + my $change = $changes{$field}; my $api_field = $api_field_names{$field} || $field; # We normalize undef to an empty string, so that the API @@ -1194,10 +1199,9 @@ sub add_comment { $bug->add_comment( $comment, { - isprivate => $params->{is_private}, - work_time => $params->{work_time}, - is_markdown => - ( defined $params->{is_markdown} ? $params->{is_markdown} : 0 ) + isprivate => $params->{is_private}, + work_time => $params->{work_time}, + is_markdown => (defined $params->{is_markdown} ? $params->{is_markdown} : 0) } ); @@ -1481,7 +1485,7 @@ sub _bug_to_hash { foreach my $attachment (@{$bug->attachments}) { next if $attachment->isprivate && !$user->is_insider; push(@result, - $self->_attachment_to_hash($attachment, $params, ['extra'], 'attachments')); + $self->_attachment_to_hash($attachment, $params, ['extra'], 'attachments')); } $item{'attachments'} = \@result; } @@ -1494,12 +1498,12 @@ sub _bug_to_hash { } if (filter_wants $params, 'comments', ['extra']) { my @result; - my $comments - = $bug->comments({order => 'oldest_to_newest', after => $params->{new_since}}); + my $comments = $bug->comments( + {order => 'oldest_to_newest', after => $params->{new_since}}); foreach my $comment (@$comments) { next if $comment->is_private && !$user->is_insider; push(@result, - $self->_translate_comment($comment, $params, ['extra'], 'comments')); + $self->_translate_comment($comment, $params, ['extra'], 'comments')); } $item{'comments'} = \@result; } @@ -1528,7 +1532,8 @@ sub _bug_to_hash { my $comment = Bugzilla::Comment->match({bug_id => $bug->id, LIMIT => 1})->[0]; $item{'description'} = ($comment && (!$comment->is_private || Bugzilla->user->is_insider)) - ? $comment->body : ''; + ? $comment->body + : ''; } if (filter_wants $params, 'dupe_of') { $item{'dupe_of'} = $self->type('int', $bug->dup_id); @@ -1549,7 +1554,7 @@ sub _bug_to_hash { = Bugzilla::Bug::GetBugActivity($bug->id, undef, $params->{new_since}, 1); foreach my $changeset (@$activity) { push(@result, - $self->_changeset_to_hash($changeset, $params, ['extra'], 'history')); + $self->_changeset_to_hash($changeset, $params, ['extra'], 'history')); } $item{'history'} = \@result; } @@ -1760,14 +1765,14 @@ sub _changeset_to_hash { my $attach_id = delete $change->{attachid}; my $comment = delete $change->{comment}; - $change->{field_name} = $self->type('string', $api_field_name); - $change->{removed} = $self->type($api_field_type, $change->{removed}); - $change->{added} = $self->type($api_field_type, $change->{added}); + $change->{field_name} = $self->type('string', $api_field_name); + $change->{removed} = $self->type($api_field_type, $change->{removed}); + $change->{added} = $self->type($api_field_type, $change->{added}); $change->{attachment_id} = $self->type('int', $attach_id) if $attach_id; $change->{comment_id} = $self->type('int', $comment->id) if $comment; $change->{comment_count} = $self->type('int', $comment->count) if $comment; - push (@{$item->{changes}}, $change); + push(@{$item->{changes}}, $change); } return filter($filters, $item, $types, $prefix); From 9cc1949e939f977e76c738a3fe074c5a6ad304ef Mon Sep 17 00:00:00 2001 From: Dylan Hardison Date: Sun, 18 Dec 2022 13:22:52 -0800 Subject: [PATCH 2/6] this changes makes api_field_* variables lazy api_field_types and api_field_names rely on values from the Bugzilla->fields definition. This in turn requires checking the schema of the bugs table at run time. That means trying to access these when a database is not configured, or before schema migrations have happened is an error. This changes those variables to hold functions (sub refs) that are comptued when called. The help performance not degrade, we make use of the Bugzilla::request_cache to only re-compute this once per request. --- Bugzilla/WebService/Bug.pm | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/Bugzilla/WebService/Bug.pm b/Bugzilla/WebService/Bug.pm index 7f90082133..1fb90f2cc5 100644 --- a/Bugzilla/WebService/Bug.pm +++ b/Bugzilla/WebService/Bug.pm @@ -110,16 +110,23 @@ use constant ATTACHMENT_MAPPED_RETURNS => { mimetype => 'content_type', }; -our %api_field_types = ( - %{{map { $_ => 'double' } Bugzilla::Bug::NUMERIC_COLUMNS()}}, - %{{map { $_ => 'dateTime' } Bugzilla::Bug::DATE_COLUMNS()}}, -); +our $api_field_types = sub { + Bugzilla::request_cache->{api_field_types} ||= { + %{{map { $_ => 'double' } Bugzilla::Bug::NUMERIC_COLUMNS()}}, + %{{map { $_ => 'dateTime' } Bugzilla::Bug::DATE_COLUMNS()}}, + }; +}; -our %api_field_names = reverse %{Bugzilla::Bug::FIELD_MAP()}; +our $api_field_names = sub { + Bugzilla::request_cache->{api_field_types} ||= do { + my $tmp = {reverse %{Bugzilla::Bug::FIELD_MAP()}}; -# This doesn't normally belong in FIELD_MAP, but we do want to translate -# "bug_group" back into "groups". -$api_field_names{'bug_group'} = 'groups'; + # This doesn't normally belong in FIELD_MAP, but we do want to translate + # "bug_group" back into "groups". + $tmp->{'bug_group'} = 'groups'; + $tmp; + }; +}; ###################################################### # Add aliases here for old method name compatibility # @@ -841,9 +848,10 @@ sub update { } my %changes = %{$all_changes{$bug->id}}; + my $names = $api_field_names->(); foreach my $field (keys %changes) { my $change = $changes{$field}; - my $api_field = $api_field_names{$field} || $field; + my $api_field = $names->{$field} || $field; # We normalize undef to an empty string, so that the API # stays consistent for things like Deadline that can become @@ -1760,8 +1768,8 @@ sub _changeset_to_hash { foreach my $change (@{$changeset->{changes}}) { my $field_name = delete $change->{fieldname}; - my $api_field_type = $api_field_types{$field_name} || 'string'; - my $api_field_name = $api_field_names{$field_name} || $field_name; + my $api_field_type = $api_field_types->()->{$field_name} || 'string'; + my $api_field_name = $api_field_names->()->{$field_name} || $field_name; my $attach_id = delete $change->{attachid}; my $comment = delete $change->{comment}; From b0135fbcfaa58f9fbe827359245c72311f1a1068 Mon Sep 17 00:00:00 2001 From: Dylan Hardison Date: Sat, 10 Sep 2022 13:33:58 -0700 Subject: [PATCH 3/6] renewed attempt at dockerization --- Dockerfile | 124 ++++++++++++++++++++++++++++------------ Dockerfile.PL | 63 ++++++++++++++++++++ Dockerfile.bmo-slim | 53 ----------------- Dockerfile.cpanfile | 19 ------ cpanfile | 111 ----------------------------------- docker-compose.test.yml | 67 ---------------------- docker-compose.yml | 121 --------------------------------------- 7 files changed, 149 insertions(+), 409 deletions(-) create mode 100755 Dockerfile.PL delete mode 100644 Dockerfile.bmo-slim delete mode 100644 Dockerfile.cpanfile delete mode 100644 cpanfile delete mode 100644 docker-compose.test.yml delete mode 100644 docker-compose.yml diff --git a/Dockerfile b/Dockerfile index 653c48fe5a..3b59196688 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,39 +1,87 @@ -FROM mozillabteam/bmo-perl-slim:20200505.1 - -ENV DEBIAN_FRONTEND noninteractive - -ARG CI -ARG CIRCLE_SHA1 -ARG CIRCLE_BUILD_URL - -ENV CI=${CI} -ENV CIRCLE_BUILD_URL=${CIRCLE_BUILD_URL} -ENV CIRCLE_SHA1=${CIRCLE_SHA1} - -ENV LOG4PERL_CONFIG_FILE=log4perl-json.conf - -RUN apt-get install -y rsync - -# we run a loopback logging server on this TCP port. -ENV LOGGING_PORT=5880 - -ENV LOCALCONFIG_ENV=1 - +# This file is generated by Dockerfile.PL. Do not edit directly. +FROM perl:5.36.0-slim AS builder WORKDIR /app - -COPY . /app - -RUN chown -R app.app /app && \ - perl -I/app -I/app/local/lib/perl5 -c -E 'use Bugzilla; BEGIN { Bugzilla->extensions }' && \ - perl -c /app/scripts/entrypoint.pl - -USER app - -RUN perl checksetup.pl --no-database --default-localconfig && \ - rm -rf /app/data /app/localconfig && \ - mkdir /app/data - -EXPOSE 8000 - -ENTRYPOINT ["/app/scripts/entrypoint.pl"] -CMD ["httpd"] +COPY Makefile.PL gen-cpanfile.pl Bugzilla.pm /app/ +COPY extensions/LimitedEmail/Config.pm /app/extensions/LimitedEmail/Config.pm +COPY extensions/LimitedEmail/disabled /app/extensions/LimitedEmail/disabled +COPY extensions/AntiSpam/Config.pm /app/extensions/AntiSpam/Config.pm +COPY extensions/AntiSpam/lib/Config.pm /app/extensions/AntiSpam/lib/Config.pm +COPY extensions/TypeSniffer/Config.pm /app/extensions/TypeSniffer/Config.pm +COPY extensions/Ember/Config.pm /app/extensions/Ember/Config.pm +COPY extensions/Ember/disabled /app/extensions/Ember/disabled +COPY extensions/RestrictComments/Config.pm /app/extensions/RestrictComments/Config.pm +COPY extensions/RestrictComments/lib/Config.pm /app/extensions/RestrictComments/lib/Config.pm +COPY extensions/ShadowBugs/Config.pm /app/extensions/ShadowBugs/Config.pm +COPY extensions/ShadowBugs/disabled /app/extensions/ShadowBugs/disabled +COPY extensions/Review/Config.pm /app/extensions/Review/Config.pm +COPY extensions/TagNewUsers/Config.pm /app/extensions/TagNewUsers/Config.pm +COPY extensions/FlagDefaultRequestee/Config.pm /app/extensions/FlagDefaultRequestee/Config.pm +COPY extensions/Push/Config.pm /app/extensions/Push/Config.pm +COPY extensions/Push/lib/Config.pm /app/extensions/Push/lib/Config.pm +COPY extensions/OpenGraph/Config.pm /app/extensions/OpenGraph/Config.pm +COPY extensions/OpenGraph/lib/Config.pm /app/extensions/OpenGraph/lib/Config.pm +COPY extensions/GitHubAuth/Config.pm /app/extensions/GitHubAuth/Config.pm +COPY extensions/GitHubAuth/lib/Config.pm /app/extensions/GitHubAuth/lib/Config.pm +COPY extensions/ContributorEngagement/Config.pm /app/extensions/ContributorEngagement/Config.pm +COPY extensions/BugmailFilter/Config.pm /app/extensions/BugmailFilter/Config.pm +COPY extensions/EditTable/Config.pm /app/extensions/EditTable/Config.pm +COPY extensions/Profanivore/Config.pm /app/extensions/Profanivore/Config.pm +COPY extensions/EditComments/Config.pm /app/extensions/EditComments/Config.pm +COPY extensions/BzAPI/Config.pm /app/extensions/BzAPI/Config.pm +COPY extensions/SecureMail/Config.pm /app/extensions/SecureMail/Config.pm +COPY extensions/SecureMail/disabled /app/extensions/SecureMail/disabled +COPY extensions/BugModal/Config.pm /app/extensions/BugModal/Config.pm +COPY extensions/InlineHistory/Config.pm /app/extensions/InlineHistory/Config.pm +COPY extensions/Bitly/Config.pm /app/extensions/Bitly/Config.pm +COPY extensions/OrangeFactor/Config.pm /app/extensions/OrangeFactor/Config.pm +COPY extensions/UserStory/Config.pm /app/extensions/UserStory/Config.pm +COPY extensions/FlagTypeComment/Config.pm /app/extensions/FlagTypeComment/Config.pm +COPY extensions/SiteMapIndex/Config.pm /app/extensions/SiteMapIndex/Config.pm +COPY extensions/Gravatar/Config.pm /app/extensions/Gravatar/Config.pm +COPY extensions/BMO/Config.pm /app/extensions/BMO/Config.pm +COPY extensions/MozProjectReview/Config.pm /app/extensions/MozProjectReview/Config.pm +COPY extensions/RequestNagger/Config.pm /app/extensions/RequestNagger/Config.pm +COPY extensions/Splinter/Config.pm /app/extensions/Splinter/Config.pm +COPY extensions/Splinter/lib/Config.pm /app/extensions/Splinter/lib/Config.pm +COPY extensions/UserProfile/Config.pm /app/extensions/UserProfile/Config.pm +COPY extensions/Voting/Config.pm /app/extensions/Voting/Config.pm +COPY extensions/GoogleAnalytics/Config.pm /app/extensions/GoogleAnalytics/Config.pm +COPY extensions/GoogleAnalytics/lib/Config.pm /app/extensions/GoogleAnalytics/lib/Config.pm +COPY extensions/Example/Config.pm /app/extensions/Example/Config.pm +COPY extensions/Example/disabled /app/extensions/Example/disabled +COPY extensions/Example/lib/Config.pm /app/extensions/Example/lib/Config.pm +COPY extensions/MyDashboard/Config.pm /app/extensions/MyDashboard/Config.pm +COPY extensions/Needinfo/Config.pm /app/extensions/Needinfo/Config.pm +COPY extensions/LastResolved/Config.pm /app/extensions/LastResolved/Config.pm +COPY extensions/GuidedBugEntry/Config.pm /app/extensions/GuidedBugEntry/Config.pm +COPY extensions/PhabBugz/Config.pm /app/extensions/PhabBugz/Config.pm +COPY extensions/PhabBugz/lib/Config.pm /app/extensions/PhabBugz/lib/Config.pm +COPY extensions/REMO/Config.pm /app/extensions/REMO/Config.pm +COPY extensions/ZPushNotify/Config.pm /app/extensions/ZPushNotify/Config.pm +COPY extensions/ProdCompSearch/Config.pm /app/extensions/ProdCompSearch/Config.pm +COPY extensions/TrackingFlags/Config.pm /app/extensions/TrackingFlags/Config.pm +COPY extensions/ComponentWatching/Config.pm /app/extensions/ComponentWatching/Config.pm +COPY extensions/Webhooks/Config.pm /app/extensions/Webhooks/Config.pm +COPY extensions/Webhooks/lib/Config.pm /app/extensions/Webhooks/lib/Config.pm +COPY extensions/OldBugMove/Config.pm /app/extensions/OldBugMove/Config.pm +COPY extensions/OldBugMove/disabled /app/extensions/OldBugMove/disabled +RUN perl Makefile.PL && \ + cpanm --notest Module::CPANfile && \ + make cpanfile && \ + cpanm --notest App::cpm && \ + apt-get update -y && \ + apt-get install -y libssl-dev libxml2-dev libexpat1-dev build-essential && \ + apt-get install -y default-libmysqlclient-dev && \ + cpm install --show-build-log-on-failure && \ + apt-get install -y apt-file && \ + apt-file update && \ + find local/lib -name '*.so' -exec ldd {} \; | awk '/=>/ {print $3}'|sort -u |xargs -IFILENAME apt-file search FILENAME | awk -F': ' '{print $1}'|sort -u > PACKAGES; +FROM perl:5.36.0-slim +WORKDIR /app +COPY --from=builder /app/PACKAGES /app/PACKAGES +RUN apt-get update -y && apt-get install -y $(cat PACKAGES) +COPY --from=builder /app/local /app/local +COPY ./ /app/ +env LOCALCONFIG_ENV 1 +ENTRYPOINT ["perl", "bugzilla.pl"] +CMD ["daemon"] diff --git a/Dockerfile.PL b/Dockerfile.PL new file mode 100755 index 0000000000..cdbc56f747 --- /dev/null +++ b/Dockerfile.PL @@ -0,0 +1,63 @@ +#!/usr/bin/env perl +use 5.10.1; +use File::Find; +use experimental 'signatures'; +use autodie; + +my $output = shift @ARGV || 'Dockerfile'; + +open my $fh, '>', $output; + +say $fh "# This file is generated by Dockerfile.PL. Do not edit directly."; +say $fh q{FROM perl:5.36.0-slim AS builder}; +say $fh q{WORKDIR /app}; +say $fh q{COPY Makefile.PL gen-cpanfile.pl Bugzilla.pm /app/}; + +find_extensions($fh); + +RUN(q{ + perl Makefile.PL && + cpanm --notest Module::CPANfile && + make cpanfile && + cpanm --notest App::cpm && + apt-get update -y && + apt-get install -y libssl-dev libxml2-dev libexpat1-dev build-essential && + apt-get install -y default-libmysqlclient-dev && + cpm install --show-build-log-on-failure && + apt-get install -y apt-file && + apt-file update && + find local/lib -name '*.so' -exec ldd {} \; | awk '/=>/ {print $3}'|sort -u |xargs -IFILENAME apt-file search FILENAME | awk -F': ' '{print $1}'|sort -u > PACKAGES; +}); + +say $fh q{FROM perl:5.36.0-slim}; +say $fh q{WORKDIR /app}; +say $fh q{COPY --from=builder /app/PACKAGES /app/PACKAGES}; +say $fh q{RUN apt-get update -y && apt-get install -y $(cat PACKAGES) }; +# several lines should be merged together, and we need to make sure to clean up +# the apt package cache after installing packages to maintain a small image size +say $fh q{COPY --from=builder /app/local /app/local}; +say $fh q{COPY ./ /app/}; + +say $fh q{env LOCALCONFIG_ENV 1}; +say $fh q{ENTRYPOINT ["perl", "bugzilla.pl"]}; +say $fh q{CMD ["daemon"]}; + +sub find_extensions($fh) { + find( + sub { + return unless -f; + return unless $_ eq 'Config.pm' or $_ eq 'disabled'; + my $path = $File::Find::name; + say $fh "COPY $path /app/$path"; + }, + 'extensions', + ); +} + +sub RUN($cmd) { + # trim leading whitespace and trailing whitespace + $cmd =~ s/^\s+|\s+$//g; + # escape newlines for Dockerfile + $cmd =~ s/\n/ \\\n/gs; + say $fh "RUN $cmd"; +} diff --git a/Dockerfile.bmo-slim b/Dockerfile.bmo-slim deleted file mode 100644 index 15c93afdd0..0000000000 --- a/Dockerfile.bmo-slim +++ /dev/null @@ -1,53 +0,0 @@ -FROM perl:5.30.2-slim AS builder - -RUN apt-get update -RUN apt-get install -y \ - apt-file \ - build-essential \ - cmake \ - curl \ - default-libmysqlclient-dev \ - git \ - libcairo-dev \ - libexpat-dev \ - libgd-dev \ - libssl-dev \ - openssl \ - zlib1g-dev - -RUN cpanm --notest --quiet App::cpm Module::CPANfile Carton::Snapshot - -WORKDIR /app - -COPY cpanfile cpanfile.snapshot /app/ - -RUN cpm install -# secure mail loop fixes -RUN cpm install http://s3.amazonaws.com/moz-devservices-bmocartons/third-party/Crypt-OpenPGP-1.15.tar.gz - -RUN apt-file update -RUN find local -name '*.so' -exec ldd {} \; \ - | egrep -v 'not.found|not.a.dynamic.executable' \ - | awk '$3 {print $3}' \ - | sort -u \ - | xargs -IFILE apt-file search -l FILE \ - | sort -u > PACKAGES - -FROM perl:5.30.2-slim - -ENV DEBIAN_FRONTEND noninteractive - -COPY --from=builder /app/local /app/local -COPY --from=builder /app/PACKAGES /app/PACKAGES - -RUN apt-get update \ - && apt-get install -y \ - curl \ - git \ - graphviz \ - libcap2-bin \ - rsync \ - $(cat /app/PACKAGES) \ - && rm -rf /var/cache/apt/* /var/lib/apt/lists/* - -RUN useradd -u 10001 -U app -m && setcap 'cap_net_bind_service=+ep' /usr/local/bin/perl diff --git a/Dockerfile.cpanfile b/Dockerfile.cpanfile deleted file mode 100644 index c5c0ad842c..0000000000 --- a/Dockerfile.cpanfile +++ /dev/null @@ -1,19 +0,0 @@ -FROM perl:5.30.2-slim - -RUN apt-get update -RUN apt-get install -y \ - build-essential curl libssl-dev zlib1g-dev openssl \ - libexpat-dev cmake git libcairo-dev libgd-dev \ - default-libmysqlclient-dev unzip wget -RUN cpanm --notest --quiet App::cpm Module::CPANfile Carton::Snapshot - -WORKDIR /app - -COPY Makefile.PL Bugzilla.pm gen-cpanfile.pl /app/ -COPY extensions/ /app/extensions/ - -RUN perl Makefile.PL -RUN make cpanfile - -RUN carton install - diff --git a/cpanfile b/cpanfile deleted file mode 100644 index 37e6c1bb75..0000000000 --- a/cpanfile +++ /dev/null @@ -1,111 +0,0 @@ -requires 'Algorithm::BloomFilter', '0.02'; -requires 'Bytes::Random::Secure'; -requires 'CGI', '4.31'; -requires 'CGI::Compile'; -requires 'CGI::Emulate::PSGI'; -requires 'CPAN::Meta::Prereqs', '2.132830'; -requires 'CPAN::Meta::Requirements', '2.121'; -requires 'Class::XSAccessor', '1.18'; -requires 'Crypt::CBC'; -requires 'Crypt::DES'; -requires 'Crypt::DES_EDE3'; -requires 'Crypt::OpenPGP', '1.12'; -requires 'Crypt::SMIME'; -requires 'DBD::mysql', '4.037'; -requires 'DBI', '1.614'; -requires 'DBIx::Class'; -requires 'DBIx::Class::Helpers', '2.034002'; -requires 'DBIx::Connector'; -requires 'Daemon::Generic'; -requires 'Date::Format', '2.23'; -requires 'Date::Parse', '2.31'; -requires 'DateTime', '0.75'; -requires 'DateTime::Format::MySQL', '0.06'; -requires 'DateTime::TimeZone', '2.11'; -requires 'Devel::NYTProf', '6.04'; -requires 'Digest::SHA', '5.47'; -requires 'EV', '4.0'; -requires 'Email::Address'; -requires 'Email::MIME', '1.904'; -requires 'Email::Send', '1.911'; -requires 'FFI::Platypus'; -requires 'File::MimeInfo::Magic'; -requires 'Future', '0.34'; -requires 'HTML::Escape', '1.10'; -requires 'HTML::Tree'; -requires 'IO::Async', '0.71'; -requires 'IO::Compress::Gzip'; -requires 'IO::Scalar'; -requires 'IPC::System::Simple'; -requires 'JSON::MaybeXS', '1.003008'; -requires 'JSON::RPC', '== 1.01'; -requires 'JSON::Validator', '3.05'; -requires 'JSON::XS', '2.0'; -requires 'LWP::Protocol::https', '6.07'; -requires 'LWP::UserAgent', '6.44'; -requires 'LWP::UserAgent::Determined'; -requires 'List::MoreUtils', '0.418'; -requires 'Log::Dispatch', '2.67'; -requires 'Log::Log4perl', '1.49'; -requires 'Math::Random::ISAAC', 'v1.0.1'; -requires 'Module::Metadata', '1.000033'; -requires 'Module::Runtime', '0.014'; -requires 'Mojo::JWT', '0.07'; -requires 'MojoX::Log::Log4perl', '0.04'; -requires 'Mojolicious', '8.42'; -requires 'Mojolicious::Plugin::ForwardedFor'; -requires 'Moo', '2.002004'; -requires 'MooX::StrictConstructor', '0.008'; -requires 'Mozilla::CA', '20160104'; -requires 'Net::DNS'; -requires 'Package::Stash', '0.37'; -requires 'Parse::CPAN::Meta', '1.44'; -requires 'PerlX::Maybe'; -requires 'Pod::Coverage::TrustPod'; -requires 'Regexp::Common'; -requires 'Role::Tiny', '2.000003'; -requires 'SOAP::Lite', '0.712'; -requires 'Scope::Guard', '0.21'; -requires 'Sereal', '4.004'; -requires 'Sub::Quote', '2.005000'; -requires 'Sys::Syslog'; -requires 'Template', '2.24'; -requires 'Test::CPAN::Meta'; -requires 'Test::Pod'; -requires 'Test::Pod::Coverage'; -requires 'Test::Taint', '1.06'; -requires 'Text::CSV_XS', '1.26'; -requires 'Text::Diff'; -requires 'Throwable', '0.200013'; -requires 'Tie::IxHash'; -requires 'Type::Tiny', '1.004004'; -requires 'URI', '1.55'; -requires 'URI::Escape::XS', '0.14'; -requires 'XMLRPC::Lite', '0.712'; -requires 'perl', '5.010001'; -requires 'version', '0.87'; -recommends 'Safe', '2.30'; - -on configure => sub { - requires 'ExtUtils::MakeMaker', '7.22'; -}; - -on build => sub { - requires 'ExtUtils::MakeMaker', '7.22'; -}; - -on test => sub { - requires 'Capture::Tiny'; - requires 'DBD::SQLite', '1.29'; - requires 'DateTime::Format::SQLite', '0.11'; - requires 'Perl::Critic::Freenode'; - requires 'Perl::Critic::Policy::Documentation::RequirePodLinksIncludeText'; - requires 'Perl::Tidy', '20180220'; - requires 'Pod::Coverage'; - requires 'Selenium::Remote::Driver', '1.31'; - requires 'Test2::V0'; - requires 'Test::More'; - requires 'Test::Perl::Critic::Progressive'; - requires 'Test::Selenium::Firefox'; - requires 'Test::WWW::Selenium'; -}; diff --git a/docker-compose.test.yml b/docker-compose.test.yml deleted file mode 100644 index de60c95894..0000000000 --- a/docker-compose.test.yml +++ /dev/null @@ -1,67 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. - -version: '3' - -services: - bmo.test: - image: bmo - command: dev_httpd - tmpfs: - - /tmp - - /run - environment: - - 'BMO_inbound_proxies=*' - - BMO_db_host=bmo.db - - BMO_db_name=bugs - - BMO_db_pass=bugs - - BMO_db_user=bugs - - BMO_memcached_namespace=bugzilla - - BMO_memcached_servers=memcached:11211 - - BMO_ses_username=ses@mozilla.bugs - - BMO_ses_password=password123456789! - - BMO_urlbase=AUTOMATIC - - BUGZILLA_ALLOW_INSECURE_HTTP=1 - - BZ_ANSWERS_FILE=/app/conf/checksetup_answers.txt - - BZ_QA_ANSWERS_FILE=/app/.circleci/checksetup_answers.txt - - BZ_QA_CONF_FILE=/app/.circleci/selenium_test.conf - - BZ_QA_CONFIG=1 - - CI=${CI} - - CIRCLE_SHA1=${CIRCLE_SHA1} - - CIRCLE_BUILD_URL=${CIRCLE_BUILD_URL} - - LOCALCONFIG_ENV=1 - - LOG4PERL_CONFIG_FILE=log4perl-test.conf - - LOGGING_PORT=5880 - - PORT=8000 - - TWD_BROWSER=firefox - - TWD_HOST=selenium - - TWD_PORT=4444 - depends_on: - - bmo.db - - memcached - - selenium - - bmo.db: - image: mysql:5.7 - tmpfs: - - /tmp - logging: - driver: "none" - environment: - - MYSQL_DATABASE=bugs - - MYSQL_USER=bugs - - MYSQL_PASSWORD=bugs - - MYSQL_ALLOW_EMPTY_PASSWORD=1 - - memcached: - image: memcached:latest - - selenium: - image: selenium/standalone-firefox:3.141.59 - shm_size: '512m' - ports: - - "5900:5900" - -volumes: - bmo-mysql-db: diff --git a/docker-compose.yml b/docker-compose.yml deleted file mode 100644 index 2bfedb8955..0000000000 --- a/docker-compose.yml +++ /dev/null @@ -1,121 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. - -version: '3.4' - -services: - bmo.test: - build: &bmo_build - context: . - dockerfile: Dockerfile - command: dev_httpd - volumes: - - bmo-data-dir:/app/data - - .:/mnt/sync - tmpfs: - - /tmp - - /run - environment: &bmo_env - - 'BMO_inbound_proxies=*' - - BMO_db_host=bmo.db - - BMO_db_name=bugs - - BMO_db_pass=bugs - - BMO_db_user=bugs - - BMO_memcached_namespace=bugzilla - - BMO_memcached_servers=memcached:11211 - - BMO_ses_username=ses@mozilla.bugs - - BMO_ses_password=password123456789! - - BMO_urlbase=http://bmo.test/ - - BUGZILLA_ALLOW_INSECURE_HTTP=1 - - BZ_ANSWERS_FILE=/app/conf/checksetup_answers.txt - - HTTP_BACKEND=morbo - - LOCALCONFIG_ENV=1 - - LOG4PERL_CONFIG_FILE=log4perl-docker.conf - - MOJO_LISTEN=http://*:80 - - PORT=80 - depends_on: - - bmo.db - - memcached - - tinyproxy - - bmo.jobqueue: - build: *bmo_build - command: jobqueue - volumes: - - bmo-data-dir:/app/data - tmpfs: - - /tmp - - /run - environment: *bmo_env - restart: always - depends_on: - - bmo.db - - memcached - - bmo.feed: - build: *bmo_build - command: - - perl - - extensions/PhabBugz/bin/phabbugz_feed.pl - - start - - '-d' - - '-f' - volumes: - - bmo-data-dir:/app/data - tmpfs: - - /tmp - - /run - environment: *bmo_env - restart: always - depends_on: - - bmo.db - - memcached - - bmo.pushd: - build: *bmo_build - command: - - perl - - extensions/Push/bin/bugzilla-pushd.pl - - start - - '-d' - - '-f' - volumes: - - bmo-data-dir:/app/data - tmpfs: - - /tmp - - /run - environment: *bmo_env - restart: always - depends_on: - - bmo.db - - memcached - - bmo.db: - image: mysql:5.7 - volumes: - - bmo-mysql-db:/var/lib/mysql - - ./docker/mysql:/etc/mysql/conf.d - tmpfs: - - /tmp - logging: - driver: "none" - environment: - - MYSQL_DATABASE=bugs - - MYSQL_USER=bugs - - MYSQL_PASSWORD=bugs - - MYSQL_ALLOW_EMPTY_PASSWORD=1 - - memcached: - image: memcached:latest - - tinyproxy: - build: - context: docker/tinyproxy - dockerfile: ./Dockerfile - ports: - - "1080:1080" - -volumes: - bmo-mysql-db: - bmo-data-dir: From b6a038d8c3b6be9b4290f7caa04d9e62ae5ed4f4 Mon Sep 17 00:00:00 2001 From: Dylan Hardison Date: Sat, 10 Sep 2022 13:56:33 -0700 Subject: [PATCH 4/6] fix missing MYMETA.json file --- Bugzilla/Install/Localconfig.pm | 2 +- Dockerfile | 1 + Dockerfile.PL | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/Bugzilla/Install/Localconfig.pm b/Bugzilla/Install/Localconfig.pm index c4c249709d..1d4f88f68b 100644 --- a/Bugzilla/Install/Localconfig.pm +++ b/Bugzilla/Install/Localconfig.pm @@ -43,7 +43,7 @@ our @EXPORT_OK = qw( ); # might want to change this for upstream -use constant ENV_PREFIX => 'BMO_'; +use constant ENV_PREFIX => 'BZ_'; use constant PARAM_OVERRIDE => qw( use_mailer_queue mail_delivery_method shadowdb shadowdbhost shadowdbport shadowdbsock ); diff --git a/Dockerfile b/Dockerfile index 3b59196688..2f72bec0a0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -81,6 +81,7 @@ WORKDIR /app COPY --from=builder /app/PACKAGES /app/PACKAGES RUN apt-get update -y && apt-get install -y $(cat PACKAGES) COPY --from=builder /app/local /app/local +COPY --from=builder /app/MYMETA.* /app/ COPY ./ /app/ env LOCALCONFIG_ENV 1 ENTRYPOINT ["perl", "bugzilla.pl"] diff --git a/Dockerfile.PL b/Dockerfile.PL index cdbc56f747..5a30d478ec 100755 --- a/Dockerfile.PL +++ b/Dockerfile.PL @@ -36,6 +36,7 @@ say $fh q{RUN apt-get update -y && apt-get install -y $(cat PACKAGES) }; # several lines should be merged together, and we need to make sure to clean up # the apt package cache after installing packages to maintain a small image size say $fh q{COPY --from=builder /app/local /app/local}; +say $fh q{COPY --from=builder /app/MYMETA.* /app/}; say $fh q{COPY ./ /app/}; say $fh q{env LOCALCONFIG_ENV 1}; From 54f100b70e273f0224919370c11d930c2fa5e5b0 Mon Sep 17 00:00:00 2001 From: Dylan Hardison Date: Sun, 18 Dec 2022 12:48:34 -0800 Subject: [PATCH 5/6] Improve error reporting from wrapped DBI methods in Bugzilla::DB --- Bugzilla/DB.pm | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/Bugzilla/DB.pm b/Bugzilla/DB.pm index 417659b203..a04a2a721c 100644 --- a/Bugzilla/DB.pm +++ b/Bugzilla/DB.pm @@ -67,8 +67,23 @@ around 'attrs' => sub { my $symbol = '&' . $method; $stash->add_symbol( $symbol => sub { - my $self = shift; - return $self->dbh->$method(@_); + my $self = shift; + my $wantarray = wantarray; + my $result = eval { + if ($wantarray) { + [$self->dbh->$method(@_)]; + } + else { + [scalar $self->dbh->$method(@_)]; + } + }; + if (not defined $result) { + my $err = $@; + Carp::confess($err); + } + else { + return $wantarray ? @$result : $result->[0]; + } } ); } @@ -511,8 +526,8 @@ sub bz_server_version { sub bz_last_key { my ($self, $table, $column) = @_; - return $self->last_insert_id(Bugzilla->localconfig->db_name, - undef, $table, $column); + return $self->last_insert_id(Bugzilla->localconfig->db_name, undef, $table, + $column); } sub bz_check_regexp { @@ -579,7 +594,7 @@ sub bz_setup_foreign_keys { # so if it doesn't have them, then we're setting up FKs # for the first time, and should be quieter about it. my $activity_fk = $self->bz_fk_info('profiles_activity', 'userid'); - my $any_fks = $activity_fk && $activity_fk->{created}; + my $any_fks = $activity_fk && $activity_fk->{created}; if (!$any_fks) { print get_text('install_fk_setup'), "\n"; } From a00fcd35efc2903cd39455b8d51fcfabf33d3332 Mon Sep 17 00:00:00 2001 From: Dylan Hardison Date: Sun, 18 Dec 2022 12:49:01 -0800 Subject: [PATCH 6/6] change default db_host to one that works with mysql --- Bugzilla/Install/Localconfig.pm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Bugzilla/Install/Localconfig.pm b/Bugzilla/Install/Localconfig.pm index 1d4f88f68b..f0040451d7 100644 --- a/Bugzilla/Install/Localconfig.pm +++ b/Bugzilla/Install/Localconfig.pm @@ -57,7 +57,7 @@ use constant LOCALCONFIG_VARS => ( {name => 'webservergroup', default => \&_sensible_group,}, {name => 'use_suexec', default => 0,}, {name => 'db_driver', default => 'mysql',}, - {name => 'db_host', default => 'localhost',}, + {name => 'db_host', default => '127.0.0.1',}, {name => 'db_name', default => 'bugs',}, {