Allow latest pandas version

Author: Hartorn

.github/workflows/build-python.yml

@@ -56,21 +56,29 @@ jobs:
         python-version: [ "3.8", "3.9", "3.10", "3.11" ]
         os: [ubuntu-latest]
         pydantic_v2: [false]
+        pandas_v1: [false]
         # https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners#supported-runners-and-hardware-resources
         include:
           - python-version: "3.10"
             os: windows-2019
             pydantic_v2: false
+            pandas_v1: false
           - python-version: "3.10"
             os: windows-2022
             pydantic_v2: false
+            pandas_v1: false
           - python-version: "3.10"
             os: macos-latest
             pydantic_v2: false
+            pandas_v1: false
           - python-version: "3.10"
             os: ubuntu-latest
             pydantic_v2: true
-
+            pandas_v1: false
+          - python-version: "3.10"
+            os: ubuntu-latest
+            pydantic_v2: false
+            pandas_v1: true
     continue-on-error: false # https://ncorti.com/blog/howto-github-actions-build-matrix
     steps:
       - name: Checkout code
@@ -112,6 +120,17 @@ jobs:
           pdm run pip freeze | grep '^pydantic'
           pdm run pip freeze | grep -q '^pydantic==${{ matrix.pydantic_v2 && '2' || '1' }}\.'
 
+      - name: Install pandas v1
+        if: ${{ matrix.pandas_v1 }}
+        run: |
+          pdm run pip uninstall pandas -y
+          pdm run pip install "pandas<2"
+
+      - name: Check Pandas installed version
+        run: |
+          pdm run pip freeze | grep '^pandas'
+          pdm run pip freeze | grep -q '^pandas==${{ matrix.pandas_v1 && '1' || '2' }}\.'
+
       - name: Test code
         env:
           PYTEST_XDIST_AUTO_NUM_WORKERS: ${{ startsWith(matrix.os,'windows-') && 1 || 2 }}

giskard/testing/tests/drift.py

@@ -83,20 +83,19 @@ def _calculate_drift_psi(actual_series, reference_series, max_categories):
     expected_distribution = expected_frequencies / len(reference_series)
     actual_distribution = actual_frequencies / len(actual_series)
     total_psi = 0
-    output_data = pd.DataFrame(columns=["Modality", "Reference_distribution", "Actual_distribution", "Psi"])
-    for category in range(len(all_modalities)):
+    output_data = []
+    for category, modality in enumerate(all_modalities):
         modality_psi = _calculate_psi(category, actual_distribution, expected_distribution)
-
         total_psi += modality_psi
-        row = {
-            "Modality": all_modalities[category],
-            "Reference_distribution": expected_distribution[category],
-            "Actual_distribution": expected_distribution[category],
-            "Psi": modality_psi,
-        }
-
-        output_data = output_data.append(pd.Series(row), ignore_index=True)
-    return total_psi, output_data
+        output_data.append(
+            {
+                "Modality": modality,
+                "Reference_distribution": expected_distribution[category],
+                "Actual_distribution": expected_distribution[category],
+                "Psi": modality_psi,
+            }
+        )
+    return total_psi, pd.DataFrame(output_data)
 
 
 def _calculate_ks(actual_series, reference_series) -> Ks_2sampResult:
@@ -131,28 +130,29 @@ def _calculate_chi_square(actual_series, reference_series, max_categories):
     # so that reference and actual has the same size
     # See https://github.com/scipy/scipy/blob/v1.8.0/scipy/stats/_stats_py.py#L6787
     k_norm = actual_series.shape[0] / reference_series.shape[0]
-    output_data = pd.DataFrame(columns=["Modality", "Reference_frequencies", "Actual_frequencies", "Chi_square"])
-    for i in range(len(all_modalities)):
+    output_data = []
+    for i, modality in enumerate(all_modalities):
         chi_square_value = (actual_frequencies[i] - expected_frequencies[i] * k_norm) ** 2 / (
             expected_frequencies[i] * k_norm
         )
         chi_square += chi_square_value
 
-        row = {
-            "Modality": all_modalities[i],
-            "Reference_frequencies": expected_frequencies[i],
-            "Actual_frequencies": actual_frequencies[i],
-            "Chi_square": chi_square_value,
-        }
+        output_data.append(
+            {
+                "Modality": modality,
+                "Reference_frequencies": expected_frequencies[i],
+                "Actual_frequencies": actual_frequencies[i],
+                "Chi_square": chi_square_value,
+            }
+        )
 
-        output_data = output_data.append(pd.Series(row), ignore_index=True)
     # if reference_series and actual_series has only one modality it turns nan (len(all_modalities)=1)
     if len(all_modalities) > 1:
         chi_cdf = chi2.cdf(chi_square, len(all_modalities) - 1)
         p_value = 1 - chi_cdf if chi_cdf != 0 else 0
     else:
         p_value = 0
-    return chi_square, p_value, output_data
+    return chi_square, p_value, pd.DataFrame(output_data)
 
 
 def _validate_feature_type(gsk_dataset, column_name, feature_type):