Feature/gsk 1233 when a test is buggy the whole test suite does not return

backend/src/main/java/ai/giskard/domain/ml/SuiteTestExecution.java

@@ -9,10 +9,7 @@
 import lombok.NoArgsConstructor;
 import lombok.Setter;
 
-import javax.persistence.Column;
-import javax.persistence.Convert;
-import javax.persistence.Entity;
-import javax.persistence.ManyToOne;
+import javax.persistence.*;
 import java.util.List;
 import java.util.Map;
 import java.util.stream.Collectors;
@@ -49,7 +46,8 @@ public class SuiteTestExecution extends BaseEntity {
     @Convert(converter = SimpleJSONStringAttributeConverter.class)
     private List<Integer> referenceSlicesSize;
 
-    private boolean passed;
+    @Enumerated(EnumType.STRING)
+    private TestResult status;
 
     @Column(columnDefinition = "VARCHAR")
     @Convert(converter = SimpleJSONStringAttributeConverter.class)
@@ -87,7 +85,9 @@ public SuiteTestExecution(SuiteTest test,
         this.unexpectedPercentNonmissing = message.getUnexpectedPercentNonmissing();
         this.partialUnexpectedIndexList = message.getPartialUnexpectedIndexListList();
         this.unexpectedIndexList = message.getUnexpectedIndexListList();
-        this.passed = message.getPassed();
+        this.status = message.getIsError()
+            ? TestResult.ERROR : message.getPassed()
+            ? TestResult.PASSED : TestResult.FAILED;
         this.metric = message.getMetric();
         this.actualSlicesSize = message.getActualSlicesSizeList();
         this.referenceSlicesSize = message.getReferenceSlicesSizeList();

backend/src/main/java/ai/giskard/web/dto/ml/SuiteTestExecutionDTO.java

@@ -1,5 +1,6 @@
 package ai.giskard.web.dto.ml;
 
+import ai.giskard.domain.ml.TestResult;
 import ai.giskard.web.dto.SuiteTestDTO;
 import com.dataiku.j2ts.annotations.UIModel;
 import lombok.Getter;
@@ -20,7 +21,7 @@ public class SuiteTestExecutionDTO {
     private List<TestResultMessageDTO> messages;
     private List<Integer> actualSlicesSize;
     private List<Integer> referenceSlicesSize;
-    private boolean passed;
+    private TestResult status;
     private List<Integer> partialUnexpectedIndexList;
     private List<Integer> unexpectedIndexList;
     private Integer missingCount;

backend/src/main/resources/config/liquibase/changelog/20230614134137_changelog.xml

@@ -0,0 +1,23 @@
+<?xml version="1.1" encoding="UTF-8" standalone="no"?>
+<databaseChangeLog xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
+                   xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+                   xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-4.1.xsd">
+    <changeSet author="kevinmessiaen (generated)" id="1686742901436-5">
+        <addColumn tableName="suite_test_execution">
+            <column name="status" type="varchar(255)"/>
+        </addColumn>
+    </changeSet>
+    <changeSet author="kevinmessiaen (generated)" id="1686742901436-6">
+        <update tableName="suite_test_execution">
+            <column name="status" value="PASSED"/>
+            <where>PASSED IS TRUE</where>
+        </update>
+        <update tableName="suite_test_execution">
+            <column name="status" value="FAILED"/>
+            <where>PASSED IS FALSE</where>
+        </update>
+    </changeSet>
+    <changeSet author="kevinmessiaen (generated)" id="1686742901436-7">
+        <dropColumn columnName="PASSED" tableName="SUITE_TEST_EXECUTION"/>
+    </changeSet>
+</databaseChangeLog>

backend/src/main/resources/config/liquibase/master.xml

@@ -24,4 +24,5 @@
     <include file="config/liquibase/changelog/20230522131655_changelog.xml" relativeToChangelogFile="false"/>
     <include file="config/liquibase/changelog/20230531151413_changelog.xml" relativeToChangelogFile="false"/>
     <include file="config/liquibase/changelog/20230531185842_changelog.xml" relativeToChangelogFile="false"/>
+    <include file="config/liquibase/changelog/20230614134137_changelog.xml" relativeToChangelogFile="false"/>
 </databaseChangeLog>

common/proto/ml-worker.proto

@@ -217,6 +217,7 @@ message TestMessage{
 
 message SingleTestResult{
   bool passed = 13;
+  bool is_error = 23;
   repeated TestMessage messages = 16;
   map<string, string> props = 14;
   float metric = 15;

frontend/src/generated-sources/ai/giskard/web/dto/ml/suite-test-execution-dto.ts

@@ -1,23 +1,24 @@
-import type {SuiteTestDTO} from './../suite-test-dto';
-import type {TestResultMessageDTO} from './test-result-message-dto';
+import type { SuiteTestDTO } from './../suite-test-dto';
+import type { TestResultMessageDTO } from './test-result-message-dto';
+import type { TestResult } from './../../../domain/ml/test-result';
 
 /**
  * Generated from ai.giskard.web.dto.ml.SuiteTestExecutionDTO
  */
 export interface SuiteTestExecutionDTO {
-    actualSlicesSize: number[];
-    inputs: {[key: string]: string};
-    messages: TestResultMessageDTO[];
-    metric: number;
-    missingCount: number;
-    missingPercent: number;
-    partialUnexpectedIndexList: number[];
-    passed: boolean;
-    referenceSlicesSize: number[];
-    test: SuiteTestDTO;
-    unexpectedCount: number;
-    unexpectedIndexList: number[];
-    unexpectedPercent: number;
-    unexpectedPercentNonmissing: number;
-    unexpectedPercentTotal: number;
-}
+  actualSlicesSize: number[];
+  inputs: { [key: string]: string };
+  messages: TestResultMessageDTO[];
+  metric: number;
+  missingCount: number;
+  missingPercent: number;
+  partialUnexpectedIndexList: number[];
+  referenceSlicesSize: number[];
+  status: TestResult;
+  test: SuiteTestDTO;
+  unexpectedCount: number;
+  unexpectedIndexList: number[];
+  unexpectedPercent: number;
+  unexpectedPercentNonmissing: number;
+  unexpectedPercentTotal: number;
+}

frontend/src/stores/test-suite.ts

@@ -5,28 +5,37 @@ import {
     JobState,
     ModelDTO,
     RequiredInputDTO,
+    TestResult,
     TestSuiteDTO,
     TestSuiteExecutionDTO
 } from '@/generated-sources';
-import {defineStore} from 'pinia';
-import {api} from '@/api';
-import {chain} from 'lodash';
-import {trackJob} from '@/utils/job-utils';
-import {useMainStore} from '@/stores/main';
+import { defineStore } from 'pinia';
+import { api } from '@/api';
+import { chain } from 'lodash';
+import { trackJob } from '@/utils/job-utils';
+import { useMainStore } from '@/stores/main';
 import mixpanel from 'mixpanel-browser';
-import {useTestSuiteCompareStore} from '@/stores/test-suite-compare';
-import {TYPE} from "vue-toastification";
-import {anonymize} from "@/utils";
+import { useTestSuiteCompareStore } from '@/stores/test-suite-compare';
+import { TYPE } from 'vue-toastification';
+import { anonymize } from '@/utils';
+
+
+function resultHasStatus(status: TestResult, result?): boolean {
+    return result !== undefined && result.status === status;
+}
 
 export const statusFilterOptions = [{
     label: 'All',
     filter: (_) => true
 }, {
     label: 'Passed',
-    filter: (result) => result !== undefined && result.passed
+    filter: (result) => resultHasStatus(TestResult.PASSED, result)
 }, {
     label: 'Failed',
-    filter: (result) => result !== undefined && !result.passed
+    filter: (result) => resultHasStatus(TestResult.FAILED, result)
+}, {
+    label: 'Error',
+    filter: (result) => resultHasStatus(TestResult.ERROR, result)
 }, {
     label: 'Not executed',
     filter: (result) => result === undefined

frontend/src/utils/tests.utils.ts

@@ -1,26 +1,59 @@
-import {statusFilterOptions} from "@/stores/test-suite";
-import {SuiteTestDTO, SuiteTestExecutionDTO} from "@/generated-sources";
+import { statusFilterOptions } from '@/stores/test-suite';
+import { SuiteTestDTO, SuiteTestExecutionDTO, TestResult } from '@/generated-sources';
+import { Colors } from '@/utils/colors';
+
+type ResultData = {
+    capitalized: string,
+    icon: string,
+    color: string,
+    textColor: string,
+    type: string
+}
+
+export const TEST_RESULT_DATA: { [testResult in TestResult]: ResultData } = {
+    [TestResult.FAILED]: {
+        capitalized: 'Failed',
+        icon: 'close',
+        color: Colors.FAIL_SURFACE,
+        textColor: Colors.ON_FAIL_SURFACE,
+        type: 'error'
+    },
+    [TestResult.PASSED]: {
+        capitalized: 'Passed',
+        icon: 'done',
+        color: Colors.PASS_SURFACE,
+        textColor: Colors.ON_PASS_SURFACE,
+        type: 'success'
+    },
+    [TestResult.ERROR]: {
+        capitalized: 'Error',
+        icon: 'mdi-alert-outline',
+        color: '#fff3cd',
+        textColor: '#856404',
+        type: 'error'
+    }
+};
 
 export class TestsUtils {
 
     static statusFilter<T extends { result?: SuiteTestExecutionDTO }>(status: string): (T) => boolean {
-        return ({result}) => statusFilterOptions.find(opt => status === opt.label)!.filter(result)
+        return ({ result }) => statusFilterOptions.find(opt => status === opt.label)!.filter(result);
     }
 
     static searchFilter<T extends { suiteTest: SuiteTestDTO }>(search: string): (T) => boolean {
-        return ({suiteTest}) => {
+        return ({ suiteTest }) => {
             const test = suiteTest.test;
 
             const keywords = search.split(' ')
-                .map(keyword => keyword.trim().toLowerCase())
-                .filter(keyword => keyword !== '');
-            
+              .map(keyword => keyword.trim().toLowerCase())
+              .filter(keyword => keyword !== '');
+
             return keywords.filter(keyword =>
-                test.name.toLowerCase().includes(keyword)
-                || test.doc?.toLowerCase()?.includes(keyword)
-                || test.displayName?.toLowerCase()?.includes(keyword)
-                || test.tags?.filter(tag => tag.includes(keyword))?.length > 0
+              test.name.toLowerCase().includes(keyword)
+              || test.doc?.toLowerCase()?.includes(keyword)
+              || test.displayName?.toLowerCase()?.includes(keyword)
+              || test.tags?.filter(tag => tag.includes(keyword))?.length > 0
             ).length === keywords.length;
-        }
+        };
     }
 }