Merge pull request #1615 from sagar118/GSK-1567-add-number-to-word-transformation

[GSK-2280] Feature: Added Number-to-Words Transformation

Author: kevinmessiaen

giskard/scanner/robustness/text_perturbation_detector.py

@@ -27,6 +27,7 @@ def _get_default_transformations(self, model: BaseModel, dataset: Dataset) -> Se
         from .text_transformations import (
             TextAccentRemovalTransformation,
             TextLowercase,
+            TextNumberToWordTransformation,
             TextPunctuationRemovalTransformation,
             TextTitleCase,
             TextTypoTransformation,
@@ -39,5 +40,6 @@ def _get_default_transformations(self, model: BaseModel, dataset: Dataset) -> Se
             TextTitleCase,
             TextTypoTransformation,
             TextPunctuationRemovalTransformation,
+            TextNumberToWordTransformation,
             TextAccentRemovalTransformation,
         ]

giskard/scanner/robustness/text_transformations.py

@@ -6,6 +6,7 @@
 
 import numpy as np
 import pandas as pd
+from num2words import num2words
 
 from ...core.core import DatasetProcessFunctionMeta
 from ...datasets import Dataset
@@ -273,6 +274,18 @@ def _switch(self, word, language):
             return None
 
 
+class TextNumberToWordTransformation(TextLanguageBasedTransformation):
+    name = "Transform numbers to words"
+
+    def _load_dictionaries(self):
+        # Regex to match numbers in text
+        self._regex = re.compile(r"(?<!\d/)(?<!\d\.)\b\d+(?:\.\d+)?\b(?!(?:\.\d+)?@|\d?/?\d)")
+
+    def make_perturbation(self, row):
+        # Replace numbers with words
+        return self._regex.sub(lambda x: num2words(x.group(), lang=row["language__gsk__meta"]), row[self.column])
+
+
 class TextReligionTransformation(TextLanguageBasedTransformation):
     name = "Switch Religion"