Handle DB errors during fixture load

[#137691589]

src/main/kotlin/io/pivotal/trilogy/application/TrilogyController.kt

@@ -2,7 +2,7 @@ package io.pivotal.trilogy.application
 
 import io.pivotal.trilogy.testproject.TestProjectBuilder
 import io.pivotal.trilogy.testproject.TestProjectResult
-import io.pivotal.trilogy.testrunner.InconsistentDatabaseException
+import io.pivotal.trilogy.testrunner.UnrecoverableException
 import io.pivotal.trilogy.testrunner.TestProjectRunner
 import org.springframework.beans.factory.annotation.Autowired
 import org.springframework.stereotype.Controller
@@ -17,8 +17,8 @@ open class TrilogyController {
         val testProject = TestProjectBuilder.build(options)
         try {
             return testProjectRunner.run(testProject)
-        } catch(e: InconsistentDatabaseException) {
-            return TestProjectResult(emptyList(), e.localizedMessage)
+        } catch(e: UnrecoverableException) {
+            return TestProjectResult(emptyList(), e.localizedMessage, fatalFailure = true)
         }
 
     }

src/main/kotlin/io/pivotal/trilogy/reporting/TestCaseReporter.kt

@@ -1,11 +1,12 @@
 package io.pivotal.trilogy.reporting
 
+import io.pivotal.trilogy.i18n.MessageCreator.getI18nMessage
 import io.pivotal.trilogy.testproject.TestProjectResult
 
 object TestCaseReporter {
     fun generateReport(result: TestProjectResult): String {
-        if (result.didFail)
-            return listOf("[FAIL] ${result.failureMessage}", "FAILED").joinToString("\n")
+        if (result.didFail or result.fatalFailure)
+            return listOf("[FAIL] ${result.failureMessage}", result.fatalFailureMessage, "FAILED").filterNotNull().joinToString("\n")
 
         return if (result.testCaseResults.all { it.didPass }) reportSuccess(result.testCaseResults) else reportFailure(result)
     }
@@ -27,6 +28,10 @@ object TestCaseReporter {
     }
     private val TestResult.displayMessage: String get() = this.errorMessage!!.prependIndent("    ")
 
+    private val TestProjectResult.fatalFailureMessage: String? get() {
+        return if (this.fatalFailure) getI18nMessage("fatalFailure") else null
+    }
+
     private val TestCaseResult.testCaseFailure: String get() {
         if (this.errorMessage == null) return ""
         return "[FAIL] ${this.testCaseName}:\n" + this.errorMessage.prependIndent("    ")

src/main/kotlin/io/pivotal/trilogy/testcase/GenericTrilogyTest.kt

@@ -1,5 +1,4 @@
 package io.pivotal.trilogy.testcase
 
 data class GenericTrilogyTest(override val description: String, val body: String,
-                              override val assertions: List<TrilogyAssertion>) : TrilogyTest {
-}
+                              override val assertions: List<TrilogyAssertion>) : TrilogyTest

src/main/kotlin/io/pivotal/trilogy/testproject/TestProjectResult.kt

@@ -2,6 +2,6 @@ package io.pivotal.trilogy.testproject
 
 import io.pivotal.trilogy.reporting.TestCaseResult
 
-data class TestProjectResult(val testCaseResults: List<TestCaseResult>, val failureMessage: String? = null) {
+data class TestProjectResult(val testCaseResults: List<TestCaseResult>, val failureMessage: String? = null, val fatalFailure: Boolean = false) {
     val didFail: Boolean get() = ! failureMessage.isNullOrBlank()
 }

src/main/kotlin/io/pivotal/trilogy/testrunner/DatabaseTestCaseRunner.kt

@@ -1,6 +1,7 @@
 package io.pivotal.trilogy.testrunner
 
 import io.pivotal.trilogy.i18n.MessageCreator.createErrorMessage
+import io.pivotal.trilogy.i18n.MessageCreator.getI18nMessage
 import io.pivotal.trilogy.reporting.TestCaseResult
 import io.pivotal.trilogy.reporting.TestResult
 import io.pivotal.trilogy.testcase.GenericTrilogyTest
@@ -22,6 +23,7 @@ class DatabaseTestCaseRunner(val testSubjectCaller: TestSubjectCaller,
             val errorMessage = missingFixtures.map { createErrorMessage("testCaseRunner.errors.missingFixture", listOf(it)) }.joinToString("\n")
             return TestCaseResult(trilogyTestCase.description, errorMessage = errorMessage)
         }
+
         trilogyTestCase.hooks.beforeAll.runSetupScripts(library)
 
         val testResults = trilogyTestCase.tests.map { test ->
@@ -79,8 +81,30 @@ class DatabaseTestCaseRunner(val testSubjectCaller: TestSubjectCaller,
 
     private fun failureWithException(e: RuntimeException) = mapOf("=FAIL=" to e.localizedMessage)
 
-    private fun List<String>.runSetupScripts(library: FixtureLibrary) = this.forEach { name -> scriptExecuter.execute(library.getSetupFixtureByName(name)) }
-    private fun List<String>.runTeardownScripts(library: FixtureLibrary) = this.forEach { name -> scriptExecuter.execute(library.getTeardownFixtureByName(name)) }
+    private fun List<String>.runSetupScripts(library: FixtureLibrary) {
+        this.forEach { name ->
+            try {
+                scriptExecuter.execute(library.getSetupFixtureByName(name))
+            } catch(e: RuntimeException) {
+                val message = createErrorMessage("testCaseRunner.errors.fixtureRun",
+                        listOf(name, getI18nMessage("vocabulary.fixtures.setup"), e.localizedMessage.prependIndent("    ")))
+                throw FixtureLoadException(message, e)
+            }
+        }
+    }
+
+    private fun List<String>.runTeardownScripts(library: FixtureLibrary) {
+        this.forEach { name ->
+            try {
+                scriptExecuter.execute(library.getTeardownFixtureByName(name))
+            } catch(e: RuntimeException) {
+                val message = createErrorMessage("testCaseRunner.errors.fixtureRun",
+                        listOf(name, getI18nMessage("vocabulary.fixtures.teardown"), e.localizedMessage.prependIndent("    ")))
+                throw FixtureLoadException(message, e)
+            }
+        }
+
+    }
 
     private fun TrilogyTest.tryProceduralTest(library: FixtureLibrary, trilogyTestCase: TrilogyTestCase): TestResult? {
         if (this !is ProcedureTrilogyTest) return null
@@ -116,6 +140,7 @@ class DatabaseTestCaseRunner(val testSubjectCaller: TestSubjectCaller,
             }
         }.filterNotNull()
     }
+
     private fun TestCaseHooks.findMissingTeardownFixtures(library: FixtureLibrary): List<String> {
         return (this.afterAll + this.afterEachTest + this.afterEachRow).map {
             try {

src/main/kotlin/io/pivotal/trilogy/testrunner/DatabaseTestProjectRunner.kt

@@ -24,13 +24,19 @@ class DatabaseTestProjectRunner(val testCaseRunner: TestCaseRunner, val scriptEx
             } catch(e: BadSqlGrammarException) {
                 val errorObjects = listOf(script.name, e.localizedMessage.prependIndent("    "))
                 val message = createErrorMessage("testProjectRunner.errors.scripts.invalid", errorObjects)
-                throw InconsistentDatabaseException(message, e)
+                throw SourceScriptLoadException(message, e)
             }
         }
     }
 
     private fun TrilogyTestProject.runTestCases(): TestProjectResult {
-        return TestProjectResult(this.testCases.map { testCaseRunner.run(it, this.fixtures) })
+        return TestProjectResult(this.testCases.map {
+            try {
+                testCaseRunner.run(it, this.fixtures)
+            } catch (e: UnrecoverableException) {
+                throw UnrecoverableException("${it.description}:\n${e.localizedMessage.prependIndent("    ")}", e)
+            }
+        })
     }
 
     private fun TrilogyTestProject.applySchema() {

src/main/kotlin/io/pivotal/trilogy/testrunner/FixtureLoadException.kt

@@ -0,0 +1,3 @@
+package io.pivotal.trilogy.testrunner
+
+class FixtureLoadException(message: String, cause: RuntimeException) : UnrecoverableException(message, cause)

src/main/kotlin/io/pivotal/trilogy/testrunner/SourceScriptLoadException.kt

@@ -0,0 +1,3 @@
+package io.pivotal.trilogy.testrunner
+
+class SourceScriptLoadException(message: String, cause: RuntimeException) : UnrecoverableException(message, cause)

src/main/kotlin/io/pivotal/trilogy/testrunner/UnrecoverableException.kt

@@ -0,0 +1,3 @@
+package io.pivotal.trilogy.testrunner
+
+open class UnrecoverableException(message: String, cause: RuntimeException) : RuntimeException(message, cause)

src/main/resources/messages.properties

@@ -8,7 +8,11 @@ testSubjectCaller.errors.mismatch.input.unexpected = Unexpected parameter(s) ''{
 output.errors.mismatch = Expected value ''{0}'' in the {1} out parameter, but received ''{2}''
 output.errors.forRow = Row {0} of {1}: {2}
 testCaseRunner.errors.missingFixture = Unable to find fixture ''{0}''
+testCaseRunner.errors.fixtureRun = Unable to load the ''{0}'' {1} fixture\n{2}
 assertionExecuter.error = Assertion failure: {0}\n{1}
 testProjectRunner.errors.scripts.invalid = Unable to load script ''{0}'':\n{1}
 connectionFailure = Unable to connect to the database:\n{0}
-applicationUsage = Usage: trilogy [<filePath>|--project=<path to trilogy test project>] --db_url=<jdbc url> --db_user=<db user name> --db_password=<db user password>\nThe db_url, db_user and db_password can be replaced by setting environment variables with the same name in upper case
+applicationUsage = Usage: trilogy [<filePath>|--project=<path to trilogy test project>] --db_url=<jdbc url> --db_user=<db user name> --db_password=<db user password>\nThe db_url, db_user and db_password can be replaced by setting environment variables with the same name in upper case
+fatalFailure = [STOP] Execution aborted - the database may be in an inconsistent state
+vocabulary.fixtures.setup = setup
+vocabulary.fixtures.teardown = teardown

src/test/kotlin/io/pivotal/trilogy/live/oracle/TrilogyControllerTests.kt

@@ -98,7 +98,10 @@ class TrilogyControllerTests : Spek({
             describe("errors") {
                 val options = TrilogyApplicationOptions(testProjectPath = "src/test/resources/projects/errors")
                 it("validates the errors") {
-                    val testCaseResult = controller.run(options).testCaseResults
+                    val result = controller.run(options)
+                    expect(false) { result.fatalFailure }
+
+                    val testCaseResult = result.testCaseResults
                     expect(false) { testCaseResult.all { it.didPass } }
                     expect(1) { testCaseResult.fold(0) { acc, result -> acc + result.passed } }
                     expect(3) { testCaseResult.fold(0) { acc, result -> acc + result.failed } }
@@ -108,7 +111,9 @@ class TrilogyControllerTests : Spek({
 
             it("fails with broken source") {
                 val options = TrilogyApplicationOptions(testProjectPath = "src/test/resources/projects/broken_source")
-                expect(true) { controller.run(options).didFail }
+                val result = controller.run(options)
+                expect(true) { result.didFail }
+                expect(true) { result.fatalFailure }
             }
 
 

src/test/kotlin/io/pivotal/trilogy/mocks/ScriptExecuterMock.kt

@@ -6,6 +6,7 @@ class ScriptExecuterMock : ScriptExecuter {
     val executeCalls: Int get() = executeArgList.count()
     val executeArgList = mutableListOf<String>()
     var shouldFailExecution = false
+    var safeExecutions: Int? = null
     var failureException = RuntimeException("SQL Script exception")
 
 
@@ -14,5 +15,14 @@ class ScriptExecuterMock : ScriptExecuter {
             throw failureException
         }
         executeArgList.add(scriptBody)
+        safeExecutions?.let { this.safeExecutions = it.dec() }
+        if (safeExecutions == 0) {
+            safeExecutions = null
+            shouldFailExecution = true
+        }
+    }
+
+    fun shouldFailAfter(safeExecutions: Int) {
+        this.safeExecutions = safeExecutions
     }
 }

src/test/kotlin/io/pivotal/trilogy/reporting/TestCaseReporterTests.kt

@@ -73,7 +73,7 @@ class TestCaseReporterTests : Spek({
 
     describe("test case failures") {
         it("should include test case failures in the report") {
-            val result = listOf(TestCaseResult("Mad test", emptyList(), "I can haz a panda"))
+            val result = listOf(TestCaseResult("Mad test", errorMessage = "I can haz a panda"))
             val report = TestCaseReporter.generateReport(TestProjectResult(result))
 
             report shouldContain "FAILED"
@@ -82,5 +82,14 @@ class TestCaseReporterTests : Spek({
         }
     }
 
+    describe("fatal failures") {
+        fit("should append the [STOP] message when a fatal failure is encountered") {
+            val result = listOf(TestCaseResult("Odd travel", errorMessage = "Walnut combines greatly with chopped steak"))
+            val report = TestCaseReporter.generateReport(TestProjectResult(result, fatalFailure = true))
+
+            report shouldContain "\n[STOP] Execution aborted - the database may be in an inconsistent state\n"
+        }
+    }
+
 
 })

src/test/kotlin/io/pivotal/trilogy/testrunner/DatabaseTestCaseRunnerTests.kt

@@ -6,6 +6,7 @@ import io.pivotal.trilogy.mocks.TestSubjectCallerStub
 import io.pivotal.trilogy.test_helpers.Fixtures
 import io.pivotal.trilogy.test_helpers.isEven
 import io.pivotal.trilogy.test_helpers.shouldContain
+import io.pivotal.trilogy.test_helpers.shouldThrow
 import io.pivotal.trilogy.testcase.GenericTrilogyTest
 import io.pivotal.trilogy.testcase.GenericTrilogyTestCase
 import io.pivotal.trilogy.testcase.ProcedureTrilogyTest
@@ -26,19 +27,28 @@ class DatabaseTestCaseRunnerTests : Spek({
     var assertionExecuterMock = AssertionExecuterMock(scriptExecuterMock)
     var testCaseRunner = DatabaseTestCaseRunner(testSubjectCallerStub, assertionExecuterMock, scriptExecuterMock)
     val testCaseHooks = TestCaseHooks()
+    val hooksWithBeforeAll = TestCaseHooks(beforeAll = listOf("set client balance"))
+    val hooksWithBeforeAllAndBeforeEachTest = TestCaseHooks(beforeAll = listOf("set client balance"),
+            beforeEachTest = listOf("Update client messages"))
+    val hooksWithBeforeEveryPossibleStep = TestCaseHooks(beforeAll = listOf("set client balance"),
+            beforeEachTest = listOf("Update client messages"), beforeEachRow = listOf("Update client balance"))
+    val hooksWithAfterEachRow = TestCaseHooks(afterEachRow = listOf("Clear client balance"))
+    val hooksWithAfterEachTest = TestCaseHooks(afterEachTest = listOf("Nowhere"))
+    val hooksWithAfterAll = TestCaseHooks(afterAll = listOf("Collision course"))
+
     val firstSetupScript = "First setup script"
     val secondSetupScript = "Second setup script"
     val thirdSetupScript = "Third setup script"
     val firstTeardownScript = "First teardown script"
     val secondTeardownScript = "Second teardown script"
     val thirdTeardownScript = "Third teardown script"
     val fixtureLibrary = FixtureLibrary(mapOf(
-            Pair("setup/set_client_balance", firstSetupScript),
-            Pair("setup/update_client_messages", secondSetupScript),
-            Pair("setup/update_client_balance", thirdSetupScript),
-            Pair("teardown/clear_client_balance", firstTeardownScript),
-            Pair("teardown/nowhere", secondTeardownScript),
-            Pair("teardown/collision_course", thirdTeardownScript)
+            "setup/set_client_balance" to firstSetupScript,
+            "setup/update_client_messages" to secondSetupScript,
+            "setup/update_client_balance" to thirdSetupScript,
+            "teardown/clear_client_balance" to firstTeardownScript,
+            "teardown/nowhere" to secondTeardownScript,
+            "teardown/collision_course" to thirdTeardownScript
     ))
 
 
@@ -378,6 +388,75 @@ class DatabaseTestCaseRunnerTests : Spek({
                     result.errorMessage!! shouldContain "Unable to find fixture '$it'"
                 }
             }
+
+            context("fixture load failure") {
+                val table = TestArgumentTable(listOf("FOO"), listOf(listOf("Bar")))
+                val proceduralTest = ProcedureTrilogyTest("Gummy stuff", table, emptyList())
+                val genericTest = GenericTrilogyTest("Fixture error test", "", emptyList())
+
+                it("throws an exception when a 'before all' fixture load fails") {
+                    scriptExecuterMock.shouldFailExecution = true
+                    val trilogyTestCase = GenericTrilogyTestCase("Fixture error", listOf(genericTest), hooksWithBeforeAll);
+                    { testCaseRunner.run(trilogyTestCase, fixtureLibrary) }.shouldThrow(FixtureLoadException::class)
+                }
+
+                it("describes the setup fixture failure in an unrecoverable error") {
+                    scriptExecuterMock.shouldFailExecution = true
+                    val trilogyTestCase = GenericTrilogyTestCase("Fixture error", listOf(genericTest), hooksWithBeforeAll)
+                    val errorMessage = try {
+                        testCaseRunner.run(trilogyTestCase, fixtureLibrary)
+                        null
+                    } catch (e: UnrecoverableException) {
+                        e.localizedMessage
+                    }
+                    expect("Unable to load the 'set client balance' setup fixture\n    SQL Script exception") { errorMessage }
+                }
+
+                it("throws and exception when a 'before each test' fixture load fails") {
+                    scriptExecuterMock.shouldFailAfter(safeExecutions = 1)
+                    val trilogyTestCase = GenericTrilogyTestCase("Before each test failure", listOf(genericTest), hooksWithBeforeAllAndBeforeEachTest);
+                    { testCaseRunner.run(trilogyTestCase, fixtureLibrary) }.shouldThrow(FixtureLoadException::class)
+                }
+
+                it("throws and exception when a 'before each row' fixture load fails") {
+                    scriptExecuterMock.shouldFailAfter(safeExecutions = 2)
+                    val trilogyTestCase = ProcedureTrilogyTestCase("DUMMY", "I failz", listOf(proceduralTest), hooksWithBeforeEveryPossibleStep);
+                    { testCaseRunner.run(trilogyTestCase, fixtureLibrary) }.shouldThrow(FixtureLoadException::class)
+                }
+
+                it("throws and exception when an 'after each row' fixture load fails") {
+                    scriptExecuterMock.shouldFailExecution = true
+                    val trilogyTestCase = ProcedureTrilogyTestCase("MUDDY", "Lol!", listOf(proceduralTest), hooksWithAfterEachRow);
+                    { testCaseRunner.run(trilogyTestCase, fixtureLibrary) }.shouldThrow(FixtureLoadException::class)
+                }
+
+                it("describes the teardown fixture failure in an unrecoverable error") {
+                    scriptExecuterMock.shouldFailExecution = true
+                    val trilogyTestCase = ProcedureTrilogyTestCase("QUOUOQ", "Sqwiwel!", listOf(proceduralTest), hooksWithAfterEachRow)
+                    val errorMessage = try {
+                        testCaseRunner.run(trilogyTestCase, fixtureLibrary)
+                        null
+                    } catch (e: UnrecoverableException) {
+                        e.localizedMessage
+                    }
+
+                    expect("Unable to load the 'Clear client balance' teardown fixture\n    SQL Script exception") { errorMessage }
+                }
+
+                it("throws an exception when 'after each test' fixture load fails") {
+                    scriptExecuterMock.shouldFailAfter(safeExecutions = 1)
+                    val trilogyTestCase = GenericTrilogyTestCase("After Each test failure", listOf(genericTest), hooksWithAfterEachTest);
+                    { testCaseRunner.run(trilogyTestCase, fixtureLibrary) }.shouldThrow(FixtureLoadException::class)
+                }
+
+                it("throws an exception when an 'after all' fixture load fails") {
+                    scriptExecuterMock.shouldFailAfter(safeExecutions = 1)
+                    val trilogyTestCase = GenericTrilogyTestCase("After all test failure", listOf(genericTest), hooksWithAfterAll);
+                    { testCaseRunner.run(trilogyTestCase, fixtureLibrary) }.shouldThrow(FixtureLoadException::class)
+
+                }
+
+            }
         }
     }
 

src/test/kotlin/io/pivotal/trilogy/testrunner/DatabaseTestProjectRunnerTests.kt

@@ -100,7 +100,7 @@ class DatabaseTestProjectRunnerTests : Spek({
             scriptExecuterMock.shouldFailExecution = true
             scriptExecuterMock.failureException = BadSqlGrammarException("Uhm...", "what?", SQLException("Bang!"))
             val runner = DatabaseTestProjectRunner(TestCaseRunnerMock(), scriptExecuterMock);
-            { runner.run(project) } shouldThrow InconsistentDatabaseException::class
+            { runner.run(project) } shouldThrow SourceScriptLoadException::class
         }
     }
 

src/test/resources/projects/broken_fixtures/src/001.validate_balance.sql

@@ -0,0 +1,8 @@
+CREATE OR REPLACE PROCEDURE VALIDATE_BALANCE(
+  V_CLIENT_ID IN NUMBER,
+  V_IS_VALID OUT NUMBER
+) AS
+BEGIN
+  SELECT CASE C.BALANCE - COALESCE(SUM(T.VALUE), 0) WHEN 0 THEN 1 ELSE 0 END INTO V_IS_VALID
+    FROM CLIENTS C LEFT JOIN TRANSACTIONS T ON T.ID_CLIENT=C.ID WHERE C.ID=V_CLIENT_ID GROUP BY C.ID, C.BALANCE;
+END;