Merge pull request #29 from vegaprotocol/24-mystery-codes

Show codes that appear in tests but not specs

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@vegaprotocol/approbation",
-  "version": "2.3.1",
+  "version": "2.3.2",
   "description": "Match Acceptance Criteria Codes with the tests that test them",
   "main": "./bin/approbation.js",
   "engine": ">= 16",

src/check-references.js

@@ -101,6 +101,9 @@ function processReferences (specs, tests) {
           criteriaWithRefs.push(c)
           criteriaReferencedTotal++
         }
+
+        // Delete used references, so that tests at the end contains only AC codes not found in specs
+        tests.delete(c)
       })
 
       if (refOutput.length > 0) {
@@ -128,7 +131,8 @@ function processReferences (specs, tests) {
   return {
     criteriaTotal,
     criteriaReferencedTotal,
-    criteriaUnreferencedTotal
+    criteriaUnreferencedTotal,
+    unknownCriteriaInTests: tests
   }
 }
 
@@ -155,21 +159,32 @@ function checkReferences (specsGlob, testsGlob, ignoreGlob) {
       }
     }
 
-    const { criteriaTotal, criteriaReferencedTotal, criteriaUnreferencedTotal } = processReferences(specs, tests)
+    const { criteriaTotal, criteriaReferencedTotal, criteriaUnreferencedTotal, unknownCriteriaInTests } = processReferences(specs, tests)
     const criteriaReferencedPercent = Math.round(criteriaReferencedTotal / criteriaTotal * 100)
     const criteriaUnreferencedPercent = Math.round(criteriaUnreferencedTotal / criteriaTotal * 100)
 
+    if (unknownCriteriaInTests.size > 0) {
+      const g = pc.bold(`${pc.red('Mystery criteria')} referenced in tests, not found in specs:`)
+      console.group(g)
+      console.dir(unknownCriteriaInTests)
+      console.groupEnd(g)
+      console.log()
+    }
+
     console.log(pc.bold('Total criteria') + `:       ${criteriaTotal}`)
     console.log(pc.green(pc.bold('With references')) + `:      ${criteriaReferencedTotal} (${criteriaReferencedPercent}%)`)
     console.log(pc.red(pc.bold('Without references')) + `:   ${criteriaUnreferencedTotal} (${criteriaUnreferencedPercent}%)`)
+    console.log(pc.red(pc.bold('Mystery criteria')) + `:     ${unknownCriteriaInTests.size}`)
+
     return {
       exitCode,
       res: {
         criteriaTotal,
         criteriaReferencedTotal,
         criteriaUnreferencedTotal,
         criteriaReferencedPercent,
-        criteriaUnreferencedPercent
+        criteriaUnreferencedPercent,
+        unknownCriteriaInTests
       }
     }
   } else {

test/check-references.test.js

@@ -103,3 +103,20 @@ test('check-references: ...which is to say both simultaneously', t => {
   t.equal(ignore.res.criteriaTotal, 1, 'Ignore: One criteria exists')
   t.equal(ignore.res.criteriaReferencedPercent, 100, 'Ignore: coverage is 100%')
 })
+
+test('check-references: detect references in tests that are not in specs', t => {
+  const path = './test/check-references/mystery-criteria/**/'
+  t.plan(4)
+
+  quiet()
+  const { res } = checkReferences(`${path}*.md`, `${path}*.feature`)
+  loud()
+
+  t.equal(res.criteriaTotal, 2, 'Two valid criteria exist in specs')
+
+  const mc = res.unknownCriteriaInTests
+  t.equal(mc.size, 1, 'There should be 1 mystery criteria')
+
+  t.equal(mc.keys().next().value, '0007-MYST-001', 'It should list the unkown code')
+  t.equal(mc.values().next().value[0], './test/check-references/mystery-criteria/tests/test.feature', 'It should point to the file with the unknown code')
+})

test/check-references/fifty-percent/tests/test.feature

@@ -1 +1 @@
-0001-VALI-001
+Test number one (0001-VALI-001)

test/check-references/mystery-criteria/specs/0001-VALI-valid.md

@@ -0,0 +1,5 @@
+This is a spec
+
+# Acceptance Criteria
+- A test (<a href="#0001-VALI-001" name="0001-VALI-001">0001-VALI-001</a>)
+- A second test (<a href="#0001-VALI-002" name="0001-VALI-002">0001-VALI-002</a>)

test/check-references/mystery-criteria/tests/test.feature

@@ -0,0 +1,2 @@
+0001-VALI-001
+0007-MYST-001