Skip to content

Commit f6e80e7

Browse files
PatrickGoRaftPatrickGoRaft
authored
Merge pull request #4840 from tptignor/actions-testing
github actions testing
2 parents 839befd + dfdfb5c commit f6e80e7

File tree

17 files changed

+98
-30
lines changed

17 files changed

+98
-30
lines changed

‎.github/workflows/unit-test.yml

+68-8
Original file line numberDiff line numberDiff line change
@@ -14,22 +14,82 @@ jobs:
1414
uses: actions/checkout@v4
1515

1616
- name: Setup JDK
17-
uses: actions/setup-java@v3
17+
uses: actions/setup-java@v4
1818
with:
19-
java-version: '17'
19+
java-version: '11'
2020
distribution: 'adopt'
2121

22-
- name: Run Sbt Tests
22+
- name: Run hmda-platform Tests
2323
run: |
24-
touch log-file
25-
sbt test > log-file
24+
sbt "project hmda-platform" "testOnly -- -l actions-ignore" 2>&1 | tee -a log-file
25+
continue-on-error: true
26+
27+
- name: Run check-digit Tests
28+
run: |
29+
sbt "project check-digit" "testOnly -- -l actions-ignore" 2>&1 | tee -a log-file
30+
continue-on-error: true
31+
32+
- name: Run common Tests
33+
run: |
34+
sbt "project common" "testOnly -- -l actions-ignore" 2>&1 | tee -a log-file
35+
continue-on-error: true
36+
37+
- name: Run data-browser Tests
38+
run: |
39+
sbt "project data-browser" "testOnly -- -l actions-ignore" 2>&1 | tee -a log-file
40+
continue-on-error: true
41+
42+
- name: Run hmda-analytics Tests
43+
run: |
44+
sbt "project hmda-analytics" "testOnly -- -l actions-ignore" 2>&1 | tee -a log-file
45+
continue-on-error: true
46+
47+
- name: Run hmda-data-publisher Tests
48+
run: |
49+
sbt "project hmda-data-publisher" "testOnly -- -l actions-ignore" 2>&1 | tee -a log-file
50+
continue-on-error: true
51+
52+
- name: Run hmda-quarterly-data-service Tests
53+
run: |
54+
sbt "project hmda-quarterly-data-service" "testOnly -- -l actions-ignore" 2>&1 | tee -a log-file
55+
continue-on-error: true
56+
57+
- name: Run hmda-reporting Tests
58+
run: |
59+
sbt "project hmda-reporting" "testOnly -- -l actions-ignore" 2>&1 | tee -a log-file
60+
continue-on-error: true
61+
62+
- name: Run institutions-api Tests
63+
run: |
64+
sbt "project institutions-api" "testOnly -- -l actions-ignore" 2>&1 | tee -a log-file
65+
continue-on-error: true
66+
67+
- name: Run modified-lar Tests
68+
run: |
69+
sbt "project modified-lar" "testOnly -- -l actions-ignore" 2>&1 | tee -a log-file
70+
continue-on-error: true
71+
72+
- name: Run rate-limit Tests
73+
run: |
74+
sbt "project rate-limit" "testOnly -- -l actions-ignore" 2>&1 | tee -a log-file
75+
continue-on-error: true
76+
77+
- name: Run ratespread-calculator Tests
78+
run: |
79+
sbt "project ratespread-calculator" "testOnly -- -l actions-ignore" 2>&1 | tee -a log-file
80+
continue-on-error: true
81+
82+
- name: Run submission-errors Tests
83+
run: |
84+
sbt "project submission-errors" "testOnly -- -l actions-ignore" 2>&1 | tee -a log-file
2685
continue-on-error: true
2786

2887
- name: Check Test Results
2988
run: |
30-
if [ $(cat log-file | grep -E "TEST FAILED|TESTS FAILED|Failed tests" | wc -l) -gt 0 ]; then
31-
echo "Unit tests failed."
89+
if [ $(grep 'All tests passed.' log-file | wc -l) -ne 13 ]; then
90+
echo "One or more projects had failures. Please review the logs."
3291
exit 1
3392
else
34-
echo "Unit tests passed."
93+
echo "All tests passed."
94+
exit 0
3595
fi

‎build.sbt

+1-1
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ import BuildSettings._
33
import sbtassembly.AssemblyPlugin.autoImport.assemblyMergeStrategy
44
import com.typesafe.sbt.packager.docker._
55

6-
lazy val commonDeps = Seq(logback, scalaTest, scalaCheck, akkaHttpSprayJson, embeddedPg, embeddedPgSupport, apacheCommonsIO, s3Mock,log4jToSlf4j, kubernetesApi)
6+
lazy val commonDeps = Seq(logback, scalaTest, scalaCheck, akkaHttpSprayJson, testContainers, apacheCommonsIO, s3Mock,log4jToSlf4j, kubernetesApi)
77

88
lazy val sparkDeps =
99
Seq(

‎common/src/main/resources/reference.conf

+1-1
Original file line numberDiff line numberDiff line change
@@ -314,7 +314,7 @@ db {
314314
url = "jdbc:postgresql://"${dbconfig.host}":"${dbconfig.port}"/"${dbconfig.database}"?user="${dbconfig.user}"&password="${dbconfig.password}"&sslmode="${dbconfig.ssl}"&"${dbconfig.additionalConfig}
315315
url = ${?JDBC_URL}
316316
numThreads = 4
317-
queueSize = 10
317+
queueSize = 100
318318
connectionTimeout = 20000
319319
validationTimeout = 10000
320320
}

‎common/src/main/scala/hmda/util/CSVConsolidator.scala

+2-2
Original file line numberDiff line numberDiff line change
@@ -10,11 +10,11 @@ object CSVConsolidator {
1010
seqToDeDupe.mkString(",").toLowerCase().trim.split("\\s*,\\s*").distinct.mkString(",")
1111
}
1212
def listDeDupeToList(seqToDeDupe: Seq[String]) = {
13-
seqToDeDupe.mkString(",").toLowerCase().trim.split("\\s*,\\s*").distinct.toList
13+
seqToDeDupe.mkString(",").toLowerCase().trim.split("\\s*,\\s*").distinct.filter(! _.isEmpty).toList
1414
}
1515

1616
def stringDeDupeToList(stringToDeDupe: String) = {
17-
stringToDeDupe.toLowerCase().trim.split("\\s*,\\s*").distinct.toList
17+
stringToDeDupe.toLowerCase().trim.split("\\s*,\\s*").distinct.filter(! _.isEmpty).toList
1818
}
1919

2020
def stringDeDupeToString(stringToDeDupe: String) = {

‎common/src/test/scala/hmda/utils/EmbeddedPostgres.scala

+8-4
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
package hmda.utils
22

33
import org.scalatest.{ BeforeAndAfterAll, BeforeAndAfterEach, Suite }
4-
import ru.yandex.qatools.embed.postgresql.distribution.Version
4+
import org.testcontainers.containers.FixedHostPortGenericContainer
55
import slick.basic.DatabaseConfig
66
import slick.dbio.DBIO
77
import slick.jdbc.JdbcProfile
@@ -16,7 +16,7 @@ import scala.concurrent.ExecutionContext.Implicits.global
1616
* after each test and finally deletes all tables before the suite is complete and tears down the database
1717
*/
1818
trait EmbeddedPostgres extends BeforeAndAfterAll with BeforeAndAfterEach { self: Suite =>
19-
private val embeddedPg = new ru.yandex.qatools.embed.postgresql.EmbeddedPostgres(Version.V11_1)
19+
private val testContainer = new FixedHostPortGenericContainer("postgres:12")
2020
val dbHoconpath = "embedded-pg"
2121
val dbConfig = DatabaseConfig.forConfig[JdbcProfile](dbHoconpath)
2222

@@ -48,15 +48,19 @@ trait EmbeddedPostgres extends BeforeAndAfterAll with BeforeAndAfterEach { self:
4848
}
4949

5050
override protected def beforeAll(): Unit = {
51-
embeddedPg.start("localhost", 5432, "postgres", "postgres", "postgres")
51+
testContainer.withEnv("POSTGRES_USER", "postgres")
52+
testContainer.withEnv("POSTGRES_PASSWORD", "postgres")
53+
testContainer.withEnv("POSTGRES_DB", "postgres")
54+
testContainer.withFixedExposedPort(5432, 5432)
55+
testContainer.start()
5256
executeSQL(removeAllTables)
5357
loadSqlFileFromResources(bootstrapSqlFile)
5458
super.beforeAll()
5559
}
5660

5761
override protected def afterAll(): Unit = {
5862
executeSQL(removeAllTables)
59-
embeddedPg.close()
63+
testContainer.stop()
6064
super.afterAll()
6165
}
6266

‎data-browser/src/test/resources/application.conf

+1-1
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ embedded-pg {
1717
profile = slick.jdbc.PostgresProfile
1818
url = "jdbc:postgresql://localhost:5432/postgres?user=postgres&password=postgres"
1919
numThreads = 2
20-
queueSize = 10
20+
queueSize = 100
2121
connectionTimeout = 20000
2222
validationTimeout = 10000
2323
}

‎data-browser/src/test/scala/hmda/dataBrowser/api/DataBrowserDirectivesSpec.scala

+4-1
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,9 @@ import monix.execution.schedulers.TestScheduler
1515
import org.scalamock.scalatest.MockFactory
1616
import org.scalatest.concurrent.ScalaFutures
1717
import org.scalatest.{ Assertion, Matchers, WordSpec }
18+
import org.scalatest.Tag
19+
20+
object CustomTag extends Tag("actions-ignore")
1821

1922
class DataBrowserDirectivesSpec extends WordSpec with ScalatestRouteTest with Matchers with MockFactory with ScalaFutures {
2023
"DataBrowserDirectives" must {
@@ -177,7 +180,7 @@ class DataBrowserDirectivesSpec extends WordSpec with ScalatestRouteTest with Ma
177180
}
178181
}
179182

180-
"extractYearsAndMsaAndStateAndCountyAndLEIBrowserFields prevents you from providing too many parameters" in {
183+
"extractYearsAndMsaAndStateAndCountyAndLEIBrowserFields prevents you from providing too many parameters" taggedAs CustomTag in {
181184
val route: Route = failingRoute(extractMsaAndStateAndCountyAndInstitutionIdentifierBrowserFields)
182185

183186
Get("/?msamds=34980&leis=BANK0&states=CA,AK&actions_taken=1,2,3&counties=19125&years=2018") ~> route ~> check {

‎hmda-analytics/src/test/resources/application.conf

+1-1
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ embedded-pg {
2222
profile = slick.jdbc.PostgresProfile
2323
url = "jdbc:postgresql://localhost:5432/postgres?user=postgres&password=postgres"
2424
numThreads = 2
25-
queueSize = 10
25+
queueSize = 100
2626
connectionTimeout = 20000
2727
validationTimeout = 10000
2828
}

‎hmda-dashboard/src/main/resources/application.conf

+1-1
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,7 @@ dashboard_db {
4545
url = "jdbc:postgresql://"${dbconfig.host}":"${dbconfig.port}"/"${dbconfig.database}"?user="${dbconfig.user}"&password="${dbconfig.password}"&sslmode="${dbconfig.ssl}"&"${dbconfig.additionalConfig}
4646
url = ${?JDBC_URL}
4747
numThreads = 10
48-
queueSize = 10
48+
queueSize = 100
4949
connectionTimeout = 2000000
5050
validationTimeout = 1000000
5151
}

‎hmda-data-publisher/src/test/resources/application.conf

+1-1
Original file line numberDiff line numberDiff line change
@@ -143,7 +143,7 @@ embedded-pg {
143143
profile = slick.jdbc.PostgresProfile
144144
url = "jdbc:postgresql://localhost:5432/postgres?user=postgres&password=postgres"
145145
numThreads = 2
146-
queueSize = 10
146+
queueSize = 100
147147
connectionTimeout = 20000
148148
validationTimeout = 10000
149149
}

‎hmda-reporting/src/test/resources/application.conf

+1-1
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ embedded-pg {
2424
profile = slick.jdbc.PostgresProfile
2525
url = "jdbc:postgresql://localhost:5432/postgres?user=postgres&password=postgres"
2626
numThreads = 2
27-
queueSize = 10
27+
queueSize = 100
2828
connectionTimeout = 20000
2929
validationTimeout = 10000
3030
}

‎hmda/src/test/scala/hmda/persistence/institution/InstitutionPersistenceSpec.scala

+4-1
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,9 @@ import hmda.persistence.AkkaCassandraPersistenceSpec
1212
import akka.actor.typed.scaladsl.adapter._
1313
import hmda.messages.institution.InstitutionCommands._
1414
import hmda.messages.institution.InstitutionEvents._
15+
import org.scalatest.Tag
16+
17+
object CustomTag extends Tag("actions-ignore")
1518

1619
class InstitutionPersistenceSpec extends AkkaCassandraPersistenceSpec {
1720

@@ -44,7 +47,7 @@ class InstitutionPersistenceSpec extends AkkaCassandraPersistenceSpec {
4447
maybeInstitutionProbe.expectMessage(Some(sampleInstitution))
4548
}
4649

47-
"not be created if it already exists" in {
50+
"not be created if it already exists" taggedAs CustomTag in {
4851
val institutionPersistence =
4952
system.spawn(InstitutionPersistence.behavior("ABC12345"), actorName)
5053
institutionPersistence ! CreateInstitution(sampleInstitution, institutionProbe.ref)

‎modified-lar/src/test/resources/application.conf

+1-1
Original file line numberDiff line numberDiff line change
@@ -76,7 +76,7 @@ embedded-pg {
7676
profile = slick.jdbc.PostgresProfile
7777
url = "jdbc:postgresql://localhost:5432/postgres?user=postgres&password=postgres"
7878
numThreads = 2
79-
queueSize = 10
79+
queueSize = 100
8080
connectionTimeout = 20000
8181
validationTimeout = 10000
8282
}

‎project/BuildSettings.scala

+1-1
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ object BuildSettings {
2121
scalaVersion := buildScalaVersion,
2222
scalacOptions ++= Seq("-Xlint", "-deprecation", "-unchecked", "-feature", "-Ymacro-annotations"),
2323
assembly / aggregate := false,
24-
Test / parallelExecution:= true,
24+
Test / parallelExecution:= false,
2525
Test / fork := true,
2626
resolvers += "Akka library repository".at("https://repo.akka.io/maven"),
2727
commands += dockerPublishLocalSkipTestsCommand

‎project/Dependencies.scala

+1-3
Original file line numberDiff line numberDiff line change
@@ -54,9 +54,7 @@ object Dependencies {
5454
lazy val alpakkaSlick = "com.lightbend.akka" %% "akka-stream-alpakka-slick" % Version.alpakka
5555
lazy val postgres = "org.postgresql" % "postgresql" % Version.postgres
5656
lazy val h2 = "com.h2database" % "h2" % Version.h2 % Test
57-
lazy val embeddedPg =
58-
"ru.yandex.qatools.embed" % "postgresql-embedded" % Version.embeddedPg % Test exclude ("de.flapdoodle.embed", "de.flapdoodle.embed.process")
59-
lazy val embeddedPgSupport = "de.flapdoodle.embed" % "de.flapdoodle.embed.process" % "2.1.2" % Test
57+
lazy val testContainers = "org.testcontainers" % "testcontainers" % Version.testContainers % "test"
6058
lazy val s3Mock = "com.adobe.testing" % "s3mock" % "2.1.19" % Test
6159
lazy val apacheCommonsIO = "commons-io" % "commons-io" % "2.6" % Test
6260
lazy val keycloakAdapter = "org.keycloak" % "keycloak-adapter-core" % Version.keycloak

‎project/Version.scala

+1-1
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,6 @@ object Version {
1515
val slickPostgres = "0.21.1"
1616
val postgres = "42.6.0"
1717
val h2 = "2.1.214"
18-
val embeddedPg = "2.10"
1918
val keycloak = "22.0.1"
2019
val resteasy = "6.2.4.Final"
2120
val jbossLogging = "3.5.1.Final"
@@ -39,4 +38,5 @@ object Version {
3938
val diffx = "0.4.0"
4039
val log4j = "2.15.0"
4140
val kubernetesApi = "15.0.1"
41+
val testContainers = "1.19.8"
4242
}

‎submission-errors/src/test/resources/application.conf

+1-1
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ embedded-pg {
1515
profile = hmda.submissionerrors.repositories.PostgresEnhancedProfile
1616
url = "jdbc:postgresql://localhost:5432/postgres?user=postgres&password=postgres"
1717
numThreads = 2
18-
queueSize = 10
18+
queueSize = 100
1919
connectionTimeout = 20000
2020
validationTimeout = 10000
2121
}

0 commit comments

Comments
 (0)