GSK-2902 GSK-2418 Add a method to execute test suite and upload result to Hub programmatically + result cleanup (#1818)

* GSK-2418 Add a method to execute test suite and upload result to Hub programmatically

* GSK-2418 Date format identical to Hub one

* GSK-2418 Updated DTO

* GSK-2418 Handle debugging

* GSK-2418 Handle debugging

* GSK-2418 Handle exception in test suite execution

* GSK-2418 Made metric optional and added test for download and upload

* GSK-2418 Updated test

* GSK-2418 Added link to the ref doc

* GSK-2418 Added todo

* GSK-2902 Cleanup TestResult information that are sent to the Hub

* GSK-2902 Updated communication tests

* Fix suite_id in _test_result_to_dto

* GSK-2418 Fixed uuid serialization

* GSK-2418 Added test

* GSK-2418 Fixed issue when sending wrong id

* GSK-2418 Fixed issue when sending wrong id

* GSK-2418 Updated save suite execution method to send configuration

* GSK-2418 Fixed issue when test has error

* GSK-2418 Fixed tests

* Fix dto format

---------

Co-authored-by: Hartorn <bazire@giskard.ai>
Co-authored-by: Hartorn <hartorn.github@gmail.com>

Author: kevinmessiaen

docs/reference/suite/index.rst

@@ -21,4 +21,6 @@ Test suite
 
 .. autoclass:: giskard.core.suite.TestSuiteResult
 
+   .. automethod:: upload
+
 .. autoclass:: giskard.core.test_result.TestResult

giskard/client/dtos.py

@@ -1,8 +1,12 @@
 from typing import Any, Dict, List, Optional
 
+from enum import Enum
+
 from pydantic import Field
 
+from giskard.core.core import TestResultStatusEnum
 from giskard.core.validation import ConfiguredBaseModel
+from giskard.utils.artifacts import serialize_parameter
 
 
 class TestInputDTO(ConfiguredBaseModel):
@@ -13,9 +17,16 @@ class TestInputDTO(ConfiguredBaseModel):
     is_alias: bool = False
     is_default_value: bool = False
 
+    @classmethod
+    def from_inputs_dict(cls, inputs: Dict[str, Any]) -> List["TestInputDTO"]:
+        return [
+            cls(name=name, value=str(serialize_parameter(value)), type=type(value).__qualname__)
+            for name, value in inputs.items()
+        ]
+
 
 class SuiteTestDTO(ConfiguredBaseModel):
-    id: int
+    id: Optional[int]
     testUuid: str
     functionInputs: Dict[str, TestInputDTO]
     displayName: Optional[str] = None
@@ -28,6 +39,56 @@ class TestSuiteDTO(ConfiguredBaseModel):
     function_inputs: List[TestInputDTO]
 
 
+class TestSuiteExecutionResult(str, Enum):
+    IN_PROGRESS = "IN_PROGRESS"
+    CANCELLED = "CANCELLED"
+    PASSED = "PASSED"
+    FAILED = "FAILED"
+    ERROR = "ERROR"
+
+
+class MLWorkerWSTestMessageType(str, Enum):
+    ERROR = "ERROR"
+    INFO = "INFO"
+
+
+class TestResultMessageDTO(ConfiguredBaseModel):
+    type: MLWorkerWSTestMessageType
+    text: str
+
+
+class SaveSuiteTestExecutionDetailsDTO(ConfiguredBaseModel):
+    inputs: Dict[str, List[str]]
+    outputs: List[str]
+    results: List[TestResultStatusEnum]
+    metadata: Dict[str, List[str]]
+
+
+class SaveSuiteTestExecutionDTO(ConfiguredBaseModel):
+    suiteTest: SuiteTestDTO
+    testUuid: str
+    displayName: str
+    inputs: Dict[str, str]
+    arguments: Dict[str, TestInputDTO]
+    messages: List[TestResultMessageDTO]
+    status: TestResultStatusEnum
+    metric: Optional[float]
+    metricName: str
+    failedIndexes: Dict[str, List[int]]
+    details: Optional[SaveSuiteTestExecutionDetailsDTO]
+
+
+class SaveSuiteExecutionDTO(ConfiguredBaseModel):
+    suiteId: Optional[int]
+    label: str
+    inputs: List[TestInputDTO]
+    result: TestSuiteExecutionResult
+    message: str
+    results: List[SaveSuiteTestExecutionDTO]
+    executionDate: str
+    completionDate: str
+
+
 class ServerInfo(ConfiguredBaseModel):
     instanceId: Optional[str] = None
     serverVersion: Optional[str] = None
@@ -51,7 +112,6 @@ class SuiteInfo(ConfiguredBaseModel):
     # The following inputs are never used and are here only for
     # coherence with backend
     id: int
-    functionInputs: List
     projectKey: str
 
 

giskard/client/giskard_client.py

@@ -15,7 +15,14 @@
 from requests_toolbelt import sessions
 
 import giskard
-from giskard.client.dtos import DatasetMetaInfo, ModelMetaInfo, ServerInfo, SuiteInfo, TestSuiteDTO
+from giskard.client.dtos import (
+    DatasetMetaInfo,
+    ModelMetaInfo,
+    SaveSuiteExecutionDTO,
+    ServerInfo,
+    SuiteInfo,
+    TestSuiteDTO,
+)
 from giskard.client.io_utils import GiskardJSONSerializer
 from giskard.client.project import Project
 from giskard.client.python_utils import warning
@@ -418,3 +425,6 @@ def save_test_suite(self, dto: TestSuiteDTO):
 
     def update_test_suite(self, suite_id: int, dto: TestSuiteDTO):
         return self._session.put(f"testing/project/{dto.project_key}/suite/{suite_id}", json=dto.dict()).json()
+
+    def save_test_suite_execution_result(self, project_key: str, dto: SaveSuiteExecutionDTO):
+        return self._session.post(f"testing/project/{project_key}/executions", json=dto.dict()).json()

giskard/core/core.py

@@ -546,3 +546,9 @@ class ComparisonClauseDTO:
     comparisonType: ComparisonType
     columnDtype: str
     value: Optional[str]
+
+
+class TestResultStatusEnum(str, Enum):
+    ERROR = "ERROR"
+    PASSED = "PASSED"
+    FAILED = "FAILED"