From 0084e99f1d2710fb0950e9c4de34c7b633ba7dd3 Mon Sep 17 00:00:00 2001
From: "don.sizemore" <don.sizemore@github.com>
Date: Wed, 3 Nov 2021 12:39:17 -0400
Subject: [PATCH 1/6] #179 enable postgres statement logging during integration
 tests

---
 defaults/main.yml                 |  1 +
 ec2/ec2-create-instance.sh        |  2 ++
 files/count.pl                    | 23 ++++++++++++++++++
 tasks/dataverse-api-testsuite.yml | 12 ++++++++++
 tasks/postgres.yml                |  1 -
 tasks/postgresql_perl_party.yml   | 39 +++++++++++++++++++++++++++++++
 tasks/postgresql_statements.yml   | 12 ++++++++++
 tests/group_vars/jenkins.yml      |  1 +
 tests/group_vars/memorytests.yml  |  1 +
 tests/group_vars/vagrant.yml      |  1 +
 10 files changed, 92 insertions(+), 1 deletion(-)
 create mode 100644 files/count.pl
 create mode 100644 tasks/postgresql_perl_party.yml
 create mode 100644 tasks/postgresql_statements.yml

diff --git a/defaults/main.yml b/defaults/main.yml
index 43d08bc..331a7c0 100644
--- a/defaults/main.yml
+++ b/defaults/main.yml
@@ -186,6 +186,7 @@ db:
     pass: dvnsecret
     jdbcurl: 
     log_lock_waits: true
+    log_statements: none
     version: 13
     port: 5432
   backups:
diff --git a/ec2/ec2-create-instance.sh b/ec2/ec2-create-instance.sh
index 29e94b4..8f34e97 100755
--- a/ec2/ec2-create-instance.sh
+++ b/ec2/ec2-create-instance.sh
@@ -255,6 +255,8 @@ if [ ! -z "$LOCAL_LOG_PATH" ]; then
    rsync -av -e "ssh -i $PEM_FILE" --ignore-missing-args $AWS_USER@$PUBLIC_DNS:/tmp/dataverse/src $LOCAL_LOG_PATH/
    # 5 server.log*
    rsync -av -e "ssh -i $PEM_FILE" --ignore-missing-args $AWS_USER@$PUBLIC_DNS:/usr/local/payara5/glassfish/domains/domain1/logs/server.log* $LOCAL_LOG_PATH/
+   # 6 query_count.out
+   rsync -av -e "ssh -i $PEM_FILE" --ignore-missing-args $AWS_USER@$PUBLIC_DNS:/tmp/query_count.out $LOCAL_LOG_PATH/
 fi
 
 # Port 8080 has been added because Ansible puts a redirect in place
diff --git a/files/count.pl b/files/count.pl
new file mode 100644
index 0000000..f41b73f
--- /dev/null
+++ b/files/count.pl
@@ -0,0 +1,23 @@
+#!/usr/bin/perl
+
+my $pglogfile = shift @ARGV;
+
+unless ( -f $pglogfile )
+{
+    die "usage: ./count.pl <PGLOGFILE>\n";
+}
+
+my $pglogfilesize = (stat($pglogfile))[7];
+print "Current size: ".$pglogfilesize." bytes.\n";
+
+system "./parse.pl < $pglogfile > tail.parsed";
+
+system "cat tail.parsed | sed 's/ where.*//' | sed 's/ WHERE.*//' | sort | uniq -c | sort -nr -k 1,2 > tail.counted";
+
+print "Parsed and counted the queries. Total number:\n";
+
+system "awk '{a+=\$1}END{print a}' < tail.counted";
+
+print "\nQueries, counted and sorted: \n\n";
+
+system "cat tail.counted";
diff --git a/tasks/dataverse-api-testsuite.yml b/tasks/dataverse-api-testsuite.yml
index bbdc2df..1b25c6c 100644
--- a/tasks/dataverse-api-testsuite.yml
+++ b/tasks/dataverse-api-testsuite.yml
@@ -6,6 +6,12 @@
 
 - include: maven.yml
 
+- include: postgresql_statements.yml
+  when:
+   - ansible_os_family == "RedHat"
+   - db.postgres.log_statements != 'none'
+   - not db.use_rds
+
 - name: do we have a pom.xml
   stat:
     path: '{{ dataverse.srcdir }}/pom.xml'
@@ -105,3 +111,9 @@
   args:
     chdir: '{{ dataverse.srcdir }}'
   when: maven.version != 'default'
+
+- include: postgresql_perl_party.yml
+  when:
+   - ansible_os_family == "RedHat"
+   - db.postgres.log_statements != 'none'
+   - not db.use_rds
diff --git a/tasks/postgres.yml b/tasks/postgres.yml
index 406d526..cfa7f04 100644
--- a/tasks/postgres.yml
+++ b/tasks/postgres.yml
@@ -56,7 +56,6 @@
   when: ansible_os_family == "Debian" and
         db.use_rds == false
 
-
 - name: install postgres client on RedHat / CentOS for RDS
   yum:
     name: 'postgresql{{ dataverse_pg_version_short }}'
diff --git a/tasks/postgresql_perl_party.yml b/tasks/postgresql_perl_party.yml
new file mode 100644
index 0000000..0229a2d
--- /dev/null
+++ b/tasks/postgresql_perl_party.yml
@@ -0,0 +1,39 @@
+---
+
+- name: "Leonid's Postgres PERL Party requires PERL"
+  ansible.builtin.package:
+    name: perl
+    state: latest
+
+# use our modified count.pl for now
+- name: "place Leonid's PERL scripts"
+  ansible.builtin.get_url:
+    url: '{{ item }}'
+    dest: '/var/lib/pgsql/{{ db.postgres.version }}/data/log/'
+    mode: '0755'
+  loop:
+    - https://raw.githubusercontent.com/IQSS/dataverse/query-counting-script/scripts/database/querycount/parse.pl
+
+- name: use our modified count.pl for now
+  copy:
+    src: count.pl
+    dest: '/var/lib/pgsql/{{ db.postgres.version }}/data/log/count.pl'
+    owner: root
+    group: root
+    mode: 0755
+
+- name: get name of Postgres logfile
+  ansible.builtin.find:
+    paths: '/var/lib/pgsql/{{ db.postgres.version }}/data/log/'
+    file_type: file
+    patterns: '*.log'
+  register: postgres_logs
+
+- name: there should be only one
+  set_fact:
+    postgres_log: "{{ postgres_logs.files | map(attribute='path') | list }}"
+
+- name: fire at will
+  ansible.builtin.shell:
+    cmd: '/usr/bin/perl count.pl {{ postgres_log }} > /tmp/query_count.out'
+    chdir: '/var/lib/pgsql/{{ db.postgres.version }}/data/log/'
diff --git a/tasks/postgresql_statements.yml b/tasks/postgresql_statements.yml
new file mode 100644
index 0000000..110c64a
--- /dev/null
+++ b/tasks/postgresql_statements.yml
@@ -0,0 +1,12 @@
+---
+
+- name: log_statements for testing
+  lineinfile:
+    path: '/var/lib/pgsql/{{ db.postgres.version }}/data/postgresql.conf'
+    regexp: '^#log_statement\ '
+    line: 'log_statement = {{ db.postgres.log_statements }}'
+
+- name: reload Postgres
+  ansible.builtin.systemd:
+    name: 'postgresql-{{ db.postgres.version }}'
+    state: reloaded
diff --git a/tests/group_vars/jenkins.yml b/tests/group_vars/jenkins.yml
index e092155..79e41ed 100644
--- a/tests/group_vars/jenkins.yml
+++ b/tests/group_vars/jenkins.yml
@@ -177,6 +177,7 @@ db:
     pass: dvnsecret
     jdbcurl:
     log_lock_waits: true
+    log_statements: all
     version: 13
     port: 5432
   backups:
diff --git a/tests/group_vars/memorytests.yml b/tests/group_vars/memorytests.yml
index ade7bd5..313a1bf 100644
--- a/tests/group_vars/memorytests.yml
+++ b/tests/group_vars/memorytests.yml
@@ -180,6 +180,7 @@ db:
     pass: dvnsecret
     jdbcurl: 
     log_lock_waits: true
+    log_statements: none
     version: 13
     port: 5432
   backups:
diff --git a/tests/group_vars/vagrant.yml b/tests/group_vars/vagrant.yml
index 7e5ade1..108ef24 100644
--- a/tests/group_vars/vagrant.yml
+++ b/tests/group_vars/vagrant.yml
@@ -182,6 +182,7 @@ db:
     pass: vagrantsekret
     jdbcurl:
     log_lock_waits: true
+    log_statements: none
     version: 13
     port: 5432
   backups:

From 0b91427b6c168923346f6a4f5e3c441f84cb6bf8 Mon Sep 17 00:00:00 2001
From: "don.sizemore" <don.sizemore@github.com>
Date: Wed, 3 Nov 2021 13:34:08 -0400
Subject: [PATCH 2/6] #179 correct postgres_log list reference

---
 tasks/postgresql_perl_party.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tasks/postgresql_perl_party.yml b/tasks/postgresql_perl_party.yml
index 0229a2d..2a4e5bb 100644
--- a/tasks/postgresql_perl_party.yml
+++ b/tasks/postgresql_perl_party.yml
@@ -35,5 +35,5 @@
 
 - name: fire at will
   ansible.builtin.shell:
-    cmd: '/usr/bin/perl count.pl {{ postgres_log }} > /tmp/query_count.out'
+    cmd: '/usr/bin/perl count.pl {{ postgres_log[0] }} > /tmp/query_count.out'
     chdir: '/var/lib/pgsql/{{ db.postgres.version }}/data/log/'

From 39a25b2a68e41ee5bb267430c771b6d8391db5d0 Mon Sep 17 00:00:00 2001
From: "don.sizemore" <don.sizemore@github.com>
Date: Thu, 4 Nov 2021 08:39:18 -0400
Subject: [PATCH 3/6] #179 update count.pl per landreev

---
 files/count.pl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/files/count.pl b/files/count.pl
index f41b73f..baf4c5a 100644
--- a/files/count.pl
+++ b/files/count.pl
@@ -12,7 +12,7 @@
 
 system "./parse.pl < $pglogfile > tail.parsed";
 
-system "cat tail.parsed | sed 's/ where.*//' | sed 's/ WHERE.*//' | sort | uniq -c | sort -nr -k 1,2 > tail.counted";
+system "cat tail.parsed | sed 's/ where.*//' | sed 's/ WHERE.*//' | sed 's/ SET .*//' | sed 's/ VALUES .*//' | sort | uniq -c | sort -nr -k 1,2 > tail.counted";
 
 print "Parsed and counted the queries. Total number:\n";
 

From 0014df346038e017fa5e32da523ada39741e64bb Mon Sep 17 00:00:00 2001
From: "don.sizemore" <don.sizemore@github.com>
Date: Tue, 30 Nov 2021 10:38:41 -0500
Subject: [PATCH 4/6] #179 Leonid added a --non-interactive switch

---
 tasks/postgresql_perl_party.yml | 13 +++----------
 1 file changed, 3 insertions(+), 10 deletions(-)

diff --git a/tasks/postgresql_perl_party.yml b/tasks/postgresql_perl_party.yml
index 2a4e5bb..eab03cb 100644
--- a/tasks/postgresql_perl_party.yml
+++ b/tasks/postgresql_perl_party.yml
@@ -5,7 +5,7 @@
     name: perl
     state: latest
 
-# use our modified count.pl for now
+# use Leonid's branch until merged into develop
 - name: "place Leonid's PERL scripts"
   ansible.builtin.get_url:
     url: '{{ item }}'
@@ -13,14 +13,7 @@
     mode: '0755'
   loop:
     - https://raw.githubusercontent.com/IQSS/dataverse/query-counting-script/scripts/database/querycount/parse.pl
-
-- name: use our modified count.pl for now
-  copy:
-    src: count.pl
-    dest: '/var/lib/pgsql/{{ db.postgres.version }}/data/log/count.pl'
-    owner: root
-    group: root
-    mode: 0755
+    - https://raw.githubusercontent.com/IQSS/dataverse/query-counting-script/scripts/database/querycount/count.pl
 
 - name: get name of Postgres logfile
   ansible.builtin.find:
@@ -35,5 +28,5 @@
 
 - name: fire at will
   ansible.builtin.shell:
-    cmd: '/usr/bin/perl count.pl {{ postgres_log[0] }} > /tmp/query_count.out'
+    cmd: '/usr/bin/perl count.pl --non-interactive {{ postgres_log[0] }} > /tmp/query_count.out'
     chdir: '/var/lib/pgsql/{{ db.postgres.version }}/data/log/'

From fad922514e8d062fd61212b8948318c1f2c95267 Mon Sep 17 00:00:00 2001
From: "don.sizemore" <don.sizemore@github.com>
Date: Wed, 1 Dec 2021 14:34:44 -0500
Subject: [PATCH 5/6] #179 use scripts from develop now that they're merged

---
 tasks/postgresql_perl_party.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tasks/postgresql_perl_party.yml b/tasks/postgresql_perl_party.yml
index eab03cb..ff8438a 100644
--- a/tasks/postgresql_perl_party.yml
+++ b/tasks/postgresql_perl_party.yml
@@ -12,8 +12,8 @@
     dest: '/var/lib/pgsql/{{ db.postgres.version }}/data/log/'
     mode: '0755'
   loop:
-    - https://raw.githubusercontent.com/IQSS/dataverse/query-counting-script/scripts/database/querycount/parse.pl
-    - https://raw.githubusercontent.com/IQSS/dataverse/query-counting-script/scripts/database/querycount/count.pl
+    - https://raw.githubusercontent.com/IQSS/dataverse/develop/scripts/database/querycount/parse.pl
+    - https://raw.githubusercontent.com/IQSS/dataverse/develop/scripts/database/querycount/count.pl
 
 - name: get name of Postgres logfile
   ansible.builtin.find:

From e445a9d101d7b4fdccf304bd16c16dc3ce72ef7c Mon Sep 17 00:00:00 2001
From: "don.sizemore" <don.sizemore@github.com>
Date: Wed, 1 Dec 2021 15:29:51 -0500
Subject: [PATCH 6/6] #179 phil correctly points out we no longer need the
 local copy of count.pl

---
 files/count.pl | 23 -----------------------
 1 file changed, 23 deletions(-)
 delete mode 100644 files/count.pl

diff --git a/files/count.pl b/files/count.pl
deleted file mode 100644
index baf4c5a..0000000
--- a/files/count.pl
+++ /dev/null
@@ -1,23 +0,0 @@
-#!/usr/bin/perl
-
-my $pglogfile = shift @ARGV;
-
-unless ( -f $pglogfile )
-{
-    die "usage: ./count.pl <PGLOGFILE>\n";
-}
-
-my $pglogfilesize = (stat($pglogfile))[7];
-print "Current size: ".$pglogfilesize." bytes.\n";
-
-system "./parse.pl < $pglogfile > tail.parsed";
-
-system "cat tail.parsed | sed 's/ where.*//' | sed 's/ WHERE.*//' | sed 's/ SET .*//' | sed 's/ VALUES .*//' | sort | uniq -c | sort -nr -k 1,2 > tail.counted";
-
-print "Parsed and counted the queries. Total number:\n";
-
-system "awk '{a+=\$1}END{print a}' < tail.counted";
-
-print "\nQueries, counted and sorted: \n\n";
-
-system "cat tail.counted";