Merge branch 'main' into feature/gsk-2574-improve-release-step

Author: vjern

docs/reference/suite/index.rst

@@ -6,6 +6,10 @@ Test suite
    .. automethod:: __init__
    .. automethod:: run
    .. automethod:: add_test
+   .. automethod:: add_test
+   .. automethod:: remove_test
+   .. automethod:: upgrade_test
+   .. automethod:: update_test_params
    .. automethod:: upload
    .. automethod:: download
 

giskard/core/suite.py

@@ -1,4 +1,4 @@
-from typing import Any, Dict, List, Optional, Tuple, Union
+from typing import Any, Callable, Dict, List, Optional, Tuple, Union
 
 import inspect
 import logging
@@ -612,6 +612,35 @@ def add_test(
 
         return self
 
+    def upgrade_test(
+        self, test: GiskardTest, migrate_params_fn: Optional[Callable[[Dict[str, Any]], Dict[str, Any]]] = None
+    ) -> "Suite":
+        """Upgrade a test with a new version, the test being upgraded are matched using display_name tests property.
+
+        Parameters
+        ----------
+        test : GiskardTest
+            The newest version of a test to be upgraded
+        migrate_params_fn : Optional[Callable[[Dict[str, Any]], Dict[str, Any]]]
+            An optional callback used to migrate the old test params into the new params
+
+        Returns
+        -------
+        Suite
+            The current instance of the test suite to allow chained calls.
+
+        """
+
+        for test_to_upgrade in self.tests:
+            if test_to_upgrade.giskard_test.display_name != test.display_name:
+                continue
+
+            test_to_upgrade.giskard_test = test
+            if migrate_params_fn is not None:
+                test_to_upgrade.provided_inputs = migrate_params_fn(test_to_upgrade.provided_inputs.copy())
+
+        return self
+
     @singledispatchmethod
     def remove_test(self, arg):
         """Remove a test from the suite.

giskard/core/test_result.py

@@ -77,7 +77,7 @@ class TestResult:
     messages: List[TestMessage] = field(default_factory=list, repr=False)
     props: Dict[str, str] = field(default_factory=dict, repr=False)
     metric_name: str = "Metric"
-    metric: float = 0
+    metric: Optional[float] = None
     missing_count: int = 0
     missing_percent: float = 0
     unexpected_count: int = 0

giskard/registry/giskard_test.py

@@ -112,6 +112,10 @@ def assert_(self):
 
             assert result.passed, message
 
+    @property
+    def display_name(self):
+        return self.meta.display_name
+
 
 Function = Callable[..., Result]
 

giskard/visualization/templates/scan_report/markdown/huggingface.md

@@ -1,16 +1,27 @@
 {% for view in groups -%}
+
 <details>
 <summary>👉{{ view.group.name }} issues ({{ view.issues|length }})</summary>
 
 {% for issue in view.issues -%}
+{{ issue.description }}
 
-| Vulnerability | Level | Data slice | Metric | Transformation | Deviation |
-|---------------|-------|------------|--------|----------------|-----------|
-| {{ view.group.name }} | <span style="color:{% if issue.level.value == "major" %} red {% else %} orange {% endif %} "> {{ issue.level.value }} {% if issue.level.value == "major" %} 🔴 {% else %} 🟡 {% endif %} </span> | {{ issue.slicing_fn if issue.slicing_fn else "—" }} | {% if "metric" in issue.meta %}{{ issue.meta.metric }} = {{ issue.meta.metric_value|format_metric }}{% else %} "—" {% endif %} | {{ issue.transformation_fn if issue.transformation_fn else "—" }} | {{ issue.meta["deviation"] if "deviation" in issue.meta else "—" }} |
+| Level | Data slice | Metric | Deviation |
+|-------|------------|--------|-----------|
+| <span style="color:{% if issue.level.value == "major" %} red {% else %} orange {% endif %} "> {{ issue.level.value }} {% if issue.level.value == "major" %} 🔴 {% else %} 🟡 {% endif %} </span> | {{ issue.slicing_fn if issue.slicing_fn else "—" }} | {% if "metric" in issue.meta %}{{ issue.meta.metric }} = {{ issue.meta.metric_value|format_metric }}{% else %} "—" {% endif %} | {{ issue.meta["deviation"] if "deviation" in issue.meta else "—" }} |
+
+{% if issue.taxonomy %}
+<h4 class="font-bold text-sm mt-4">Taxonomy</h4>
+{% for tag in issue.taxonomy %}
+<span class="inline-block bg-blue-300/25 text-zinc-100 px-2 py-0.5 rounded-sm text-sm mr-1 my-2">
+    {{ tag }}
+</span>
+{% endfor %}
+<br />
+{% endif %}
 
 <details>
 <summary> 🔍✨Examples</summary>
-{{ issue.description }}
 
 {% if issue.examples(3)|length %}
 {{ issue.examples(issue.meta.num_examples if "num_examples" in issue.meta else 3).to_markdown(
@@ -22,5 +33,7 @@ else True)|replace("\\n", "<br>")|safe }}
 {% endfor %}
 
 </details>
+<!-- line breaker -->
 {% endfor -%}
 <br />
+

pyproject.toml

@@ -132,7 +132,7 @@ giskard = "giskard.integrations.mlflow.giskard_evaluator:GiskardEvaluator"
 name = "giskard"
 readme = "README.md"
 license = { text = "Apache Software License 2.0" }
-version = "2.6.0"
+version = "2.7.0"
 description = "The testing framework dedicated to ML models, from tabular to LLMs"
 authors = [{ name = "Giskard AI", email = "hello@giskard.ai" }]
 keywords = ["Artificial Intelligence", "Machine Learning", "Quality", "MLOps"]

tests/test_programmatic.py

@@ -307,3 +307,81 @@ def test_with_custom_types():
 
     result = _test_with_custom_types(b=CustomClass(2), a=CustomClass(3)).execute()
     assert result
+
+
+def test_upgrade_test_with_migration():
+    @test(name="Named test")
+    def my_named_test(is_pass: bool):
+        return is_pass
+
+    suite = Suite().add_test(my_named_test(True), "True").add_test(my_named_test(False), "False")
+
+    assert len(suite.tests) == 2
+    assert suite.tests[0].test_id == "True"
+    assert suite.tests[0].giskard_test.meta.uuid == my_named_test.meta.uuid
+    assert suite.tests[0].provided_inputs == {"is_pass": True}
+    assert suite.tests[1].test_id == "False"
+    assert suite.tests[1].giskard_test.meta.uuid == my_named_test.meta.uuid
+    assert suite.tests[1].provided_inputs == {"is_pass": False}
+
+    @test(name="Named test")
+    def my_named_test_v2(passed: bool):
+        return passed
+
+    suite.upgrade_test(my_named_test_v2, lambda params: {"passed": params["is_pass"]})
+
+    assert len(suite.tests) == 2
+    assert suite.tests[0].test_id == "True"
+    assert suite.tests[0].giskard_test.meta.uuid == my_named_test_v2.meta.uuid
+    assert suite.tests[0].provided_inputs == {"passed": True}
+    assert suite.tests[1].test_id == "False"
+    assert suite.tests[1].giskard_test.meta.uuid == my_named_test_v2.meta.uuid
+    assert suite.tests[1].provided_inputs == {"passed": False}
+
+
+def test_upgrade_test_without():
+    @test(name="Named test")
+    def my_named_test(is_pass: bool):
+        return is_pass
+
+    suite = Suite().add_test(my_named_test(True), "True")
+
+    assert len(suite.tests) == 1
+    assert suite.tests[0].test_id == "True"
+    assert suite.tests[0].giskard_test.meta.uuid == my_named_test.meta.uuid
+    assert suite.tests[0].provided_inputs == {"is_pass": True}
+
+    @test(name="Named test")
+    def my_named_test_v2(passed: bool):
+        return passed
+
+    suite.upgrade_test(my_named_test_v2)
+
+    assert len(suite.tests) == 1
+    assert suite.tests[0].test_id == "True"
+    assert suite.tests[0].giskard_test.meta.uuid == my_named_test_v2.meta.uuid
+    assert suite.tests[0].provided_inputs == {"is_pass": True}
+
+
+def test_upgrade_test_not_matching():
+    @test(name="Named test")
+    def my_named_test(is_pass: bool):
+        return is_pass
+
+    suite = Suite().add_test(my_named_test(True), "True")
+
+    assert len(suite.tests) == 1
+    assert suite.tests[0].test_id == "True"
+    assert suite.tests[0].giskard_test.meta.uuid == my_named_test.meta.uuid
+    assert suite.tests[0].provided_inputs == {"is_pass": True}
+
+    @test(name="Named test V2")
+    def my_named_test_v2(passed: bool):
+        return passed
+
+    suite.upgrade_test(my_named_test_v2)
+
+    assert len(suite.tests) == 1
+    assert suite.tests[0].test_id == "True"
+    assert suite.tests[0].giskard_test.meta.uuid == my_named_test.meta.uuid
+    assert suite.tests[0].provided_inputs == {"is_pass": True}