GSK-2788 Refactored artifact upload/download to not be dependant of project (#1797)

* GSK-2788 Problem after migrating to 2.7.0

* GSK-2788 Fixed inspection upload and updated test_ml_worker.py

* GSK-2788 Updated expected artifact upload URL to remove project_key

* GSK-2788 Updated expected artifact upload URL to remove project_key

* GSK-2788 Updated expected artifact upload URL to remove project_key

* GSK-2788 Updated expected artifact upload URL to remove project_key

---------

Co-authored-by: Hartorn <bazire@giskard.ai>

Author: kevinmessiaen

giskard/core/savable.py

@@ -101,14 +101,14 @@ def upload(
 
         name = self._get_name()
 
-        local_dir = settings.home_dir / settings.cache_dir / (project_key or "global") / name / self.meta.uuid
+        local_dir = settings.home_dir / settings.cache_dir / name / self.meta.uuid
 
         if not local_dir.exists():
             os.makedirs(local_dir)
         self.save(local_dir)
         logger.debug(f"Saved {name}.{self.meta.uuid}")
 
-        client.log_artifacts(local_dir, posixpath.join(project_key or "global", self._get_name(), self.meta.uuid))
+        client.log_artifacts(local_dir, posixpath.join(self._get_name(), self.meta.uuid))
         self.meta = client.save_meta(self._get_meta_endpoint(self.meta.uuid, project_key), self.meta)
 
         return self.meta.uuid
@@ -134,7 +134,7 @@ def download(cls, uuid: str, client: Optional[GiskardClient], project_key: Optio
         """
         name = cls._get_name()
 
-        local_dir = settings.home_dir / settings.cache_dir / (project_key or "global") / name / uuid
+        local_dir = settings.home_dir / settings.cache_dir / name / uuid
 
         if client is None:
             meta = cls._load_meta_locally(local_dir, uuid)
@@ -148,7 +148,7 @@ def download(cls, uuid: str, client: Optional[GiskardClient], project_key: Optio
 
         if data is None:
             assert client is not None, f"Cannot find existing {name} {uuid}"
-            client.load_artifact(local_dir, posixpath.join(project_key or "global", name, uuid))
+            client.load_artifact(local_dir, posixpath.join(name, uuid))
             data = cls.load(local_dir, uuid, meta)
 
         return data

giskard/datasets/base/__init__.py

@@ -497,7 +497,7 @@ def upload(self, client: GiskardClient, project_key: str):
 
         with tempfile.TemporaryDirectory(prefix="giskard-dataset-") as local_path:
             original_size_bytes, compressed_size_bytes = self.save(Path(local_path), dataset_id)
-            client.log_artifacts(local_path, posixpath.join(project_key, "datasets", dataset_id))
+            client.log_artifacts(local_path, posixpath.join("datasets", dataset_id))
             client.save_dataset_meta(
                 project_key,
                 dataset_id,
@@ -573,7 +573,7 @@ def download(cls, client: Optional[GiskardClient], project_key, dataset_id, samp
         Returns:
             Dataset: A Dataset object that represents the downloaded dataset.
         """
-        local_dir = settings.home_dir / settings.cache_dir / project_key / "datasets" / dataset_id
+        local_dir = settings.home_dir / settings.cache_dir / "datasets" / dataset_id
 
         if client is None:
             # internal worker case, no token based http client
@@ -589,7 +589,7 @@ def download(cls, client: Optional[GiskardClient], project_key, dataset_id, samp
                     category_features=saved_meta["category_features"],
                 )
         else:
-            client.load_artifact(local_dir, posixpath.join(project_key, "datasets", dataset_id))
+            client.load_artifact(local_dir, posixpath.join("datasets", dataset_id))
             meta: DatasetMeta = client.load_dataset_meta(project_key, dataset_id)
 
         df = cls.load(local_dir / get_file_name("data", "csv.zst", sample))

giskard/ml_worker/websocket/listener.py

@@ -354,25 +354,25 @@ def run_model(client: Optional[GiskardClient], params: websocket.RunModelParam,
         if client:
             client.log_artifact(
                 tmp_dir / predictions_csv,
-                f"{params.project_key}/models/inspections/{params.inspectionId}",
+                f"models/inspections/{params.inspectionId}",
             )
         else:
             log_artifact_local(
                 tmp_dir / predictions_csv,
-                f"{params.project_key}/models/inspections/{params.inspectionId}",
+                f"models/inspections/{params.inspectionId}",
             )
 
         calculated_csv = get_file_name("calculated", "csv", params.dataset.sample)
         calculated.to_csv(index=False, path_or_buf=tmp_dir / calculated_csv)
         if client:
             client.log_artifact(
                 tmp_dir / calculated_csv,
-                f"{params.project_key}/models/inspections/{params.inspectionId}",
+                f"models/inspections/{params.inspectionId}",
             )
         else:
             log_artifact_local(
                 tmp_dir / calculated_csv,
-                f"{params.project_key}/models/inspections/{params.inspectionId}",
+                f"models/inspections/{params.inspectionId}",
             )
     return websocket.Empty()
 

giskard/ml_worker/websocket/utils.py

@@ -34,7 +34,7 @@
 )
 from giskard.ml_worker.websocket.action import MLWorkerAction
 from giskard.models.base import BaseModel
-from giskard.path_utils import projects_dir
+from giskard.path_utils import artifacts_dir
 from giskard.registry.registry import tests_registry
 from giskard.registry.slicing_function import SlicingFunction
 from giskard.registry.transformation_function import TransformationFunction
@@ -133,9 +133,9 @@ def log_artifact_local(local_file, artifact_path=None):
     file_name = os.path.basename(local_file)
 
     if artifact_path:
-        artifact_file = projects_dir / artifact_path / file_name
+        artifact_file = artifacts_dir / artifact_path / file_name
     else:
-        artifact_file = projects_dir / file_name
+        artifact_file = artifacts_dir / file_name
     artifact_file.parent.mkdir(parents=True, exist_ok=True)
 
     shutil.copy(local_file, artifact_file)

giskard/models/base/model.py

@@ -459,7 +459,7 @@ def upload(self, client: GiskardClient, project_key, validate_ds=None, *_args, *
             self.save(f)
 
             if client is not None:
-                client.log_artifacts(f, posixpath.join(project_key, "models", str(self.id)))
+                client.log_artifacts(f, posixpath.join("models", str(self.id)))
                 client.save_model_meta(project_key, self.id, self.meta, platform.python_version(), get_size(f))
         return str(self.id)
 
@@ -479,13 +479,13 @@ def download(cls, client: Optional[GiskardClient], project_key, model_id, *_args
         Raises:
             AssertionError: If the local directory where the model should be saved does not exist.
         """
-        local_dir = settings.home_dir / settings.cache_dir / project_key / "models" / model_id
+        local_dir = settings.home_dir / settings.cache_dir / "models" / model_id
         if client is None:
             # internal worker case, no token based http client [deprecated, to be removed]
             assert local_dir.exists(), f"Cannot find existing model {project_key}.{model_id} in {local_dir}"
             meta_response, meta = cls.read_meta_from_local_dir(local_dir)
         else:
-            client.load_artifact(local_dir, posixpath.join(project_key, "models", model_id))
+            client.load_artifact(local_dir, posixpath.join("models", model_id))
             meta_response: ModelMetaInfo = client.load_model_meta(project_key, model_id)
             # internal worker case, no token based http client
             if not local_dir.exists():

giskard/path_utils.py

@@ -3,15 +3,7 @@
 from giskard.settings import settings
 
 run_dir = settings.home_dir / "run"
-projects_dir = settings.home_dir / "projects"
-
-
-def model_path(project_key: str, file_name: str) -> Path:
-    return projects_dir / project_key / "models" / file_name
-
-
-def dataset_path(project_key: str, file_name: str) -> Path:
-    return projects_dir / project_key / "datasets" / file_name
+artifacts_dir = settings.home_dir / "artifacts"
 
 
 def get_size(path: str):

tests/communications/test_websocket_actor.py

@@ -131,10 +131,10 @@ def test_websocket_actor_run_model_internal(data, model, sample, request):
     project_key = str(uuid.uuid4())  # Use a UUID to separate the resources used by the tests
     inspection_id = 0
 
-    with utils.MockedProjectCacheDir(project_key):
+    with utils.MockedProjectCacheDir():
         # Prepare dataset and model
-        utils.local_save_model_under_giskard_home_cache(model, project_key)
-        utils.local_save_dataset_under_giskard_home_cache(dataset, project_key)
+        utils.local_save_model_under_giskard_home_cache(model)
+        utils.local_save_dataset_under_giskard_home_cache(dataset)
 
         params = websocket.RunModelParam(
             model=websocket.ArtifactRef(project_key=project_key, id=str(model.id)),
@@ -146,7 +146,7 @@ def test_websocket_actor_run_model_internal(data, model, sample, request):
         reply = listener.run_model(client=None, params=params)
         assert isinstance(reply, websocket.Empty)
         # Inspection are logged locally
-        inspection_path = settings.home_dir / "projects" / project_key / "models" / "inspections" / str(inspection_id)
+        inspection_path = settings.home_dir / "artifacts" / "models" / "inspections" / str(inspection_id)
         assert (inspection_path / get_file_name("predictions", "csv", sample)).exists()
         assert (inspection_path / get_file_name("calculated", "csv", sample)).exists()
         # Clean up
@@ -170,7 +170,7 @@ def test_websocket_actor_run_model(data, model, sample, request):
     project_key = str(uuid.uuid4())  # Use a UUID to separate the resources used by the tests
     inspection_id = 0
 
-    with utils.MockedProjectCacheDir(project_key):
+    with utils.MockedProjectCacheDir():
         params = websocket.RunModelParam(
             model=websocket.ArtifactRef(project_key=project_key, id=str(model.id)),
             dataset=websocket.ArtifactRef(project_key=project_key, id=str(dataset.id), sample=sample),
@@ -199,10 +199,10 @@ def test_websocket_actor_run_model_for_data_frame_regression(internal, request):
 
     project_key = str(uuid.uuid4())  # Use a UUID to separate the resources used by the tests
 
-    with utils.MockedProjectCacheDir(project_key), utils.MockedClient(mock_all=False) as (client, mr):
+    with utils.MockedProjectCacheDir(), utils.MockedClient(mock_all=False) as (client, mr):
         # Prepare model
         if internal:
-            utils.local_save_model_under_giskard_home_cache(model, project_key)
+            utils.local_save_model_under_giskard_home_cache(model)
         else:
             utils.register_uri_for_model_meta_info(mr, model, project_key)
             utils.register_uri_for_model_artifact_info(mr, model, project_key, register_file_contents=True)
@@ -239,10 +239,10 @@ def test_websocket_actor_run_model_for_data_frame_classification(internal, reque
 
     project_key = str(uuid.uuid4())  # Use a UUID to separate the resources used by the tests
 
-    with utils.MockedProjectCacheDir(project_key), utils.MockedClient(mock_all=False) as (client, mr):
+    with utils.MockedProjectCacheDir(), utils.MockedClient(mock_all=False) as (client, mr):
         # Prepare model
         if internal:
-            utils.local_save_model_under_giskard_home_cache(model, project_key)
+            utils.local_save_model_under_giskard_home_cache(model)
         else:
             utils.register_uri_for_model_meta_info(mr, model, project_key)
             utils.register_uri_for_model_artifact_info(mr, model, project_key, register_file_contents=True)
@@ -279,10 +279,10 @@ def test_websocket_actor_explain_ws_internal(data, model, request):
 
     project_key = str(uuid.uuid4())  # Use a UUID to separate the resources used by the tests
 
-    with utils.MockedProjectCacheDir(project_key):
+    with utils.MockedProjectCacheDir():
         # Prepare model and dataset
-        utils.local_save_model_under_giskard_home_cache(model, project_key)
-        utils.local_save_dataset_under_giskard_home_cache(dataset, project_key)
+        utils.local_save_model_under_giskard_home_cache(model)
+        utils.local_save_dataset_under_giskard_home_cache(dataset)
 
         params = websocket.ExplainParam(
             model=websocket.ArtifactRef(project_key=project_key, id=str(model.id)),
@@ -301,7 +301,7 @@ def test_websocket_actor_explain_ws(data, model, request):
 
     project_key = str(uuid.uuid4())  # Use a UUID to separate the resources used by the tests
 
-    with utils.MockedProjectCacheDir(project_key), utils.MockedClient(mock_all=False) as (client, mr):
+    with utils.MockedProjectCacheDir(), utils.MockedClient(mock_all=False) as (client, mr):
         # Prepare model and dataset
         utils.register_uri_for_model_meta_info(mr, model, project_key)
         utils.register_uri_for_model_artifact_info(mr, model, project_key, register_file_contents=True)
@@ -324,10 +324,10 @@ def test_websocket_actor_explain_text_ws_regression(internal, request):
 
     project_key = str(uuid.uuid4())  # Use a UUID to separate the resources used by the tests
 
-    with utils.MockedProjectCacheDir(project_key), utils.MockedClient(mock_all=False) as (client, mr):
+    with utils.MockedProjectCacheDir(), utils.MockedClient(mock_all=False) as (client, mr):
         # Prepare model and dataset
         if internal:
-            utils.local_save_model_under_giskard_home_cache(model, project_key)
+            utils.local_save_model_under_giskard_home_cache(model)
         else:
             utils.register_uri_for_model_meta_info(mr, model, project_key)
             utils.register_uri_for_model_artifact_info(mr, model, project_key, register_file_contents=True)
@@ -357,9 +357,9 @@ def test_websocket_actor_explain_text_ws_not_text(request):
 
     project_key = str(uuid.uuid4())  # Use a UUID to separate the resources used by the tests
 
-    with utils.MockedProjectCacheDir(project_key), utils.MockedClient(mock_all=False) as (client, mr):
+    with utils.MockedProjectCacheDir(), utils.MockedClient(mock_all=False) as (client, mr):
         # Prepare model and dataset
-        utils.local_save_model_under_giskard_home_cache(model, project_key)
+        utils.local_save_model_under_giskard_home_cache(model)
 
         not_text_feature_name = None
         for col_name, col_type in dataset.column_types.items():
@@ -385,10 +385,10 @@ def test_websocket_actor_explain_text_ws_classification(internal, request):
 
     project_key = str(uuid.uuid4())  # Use a UUID to separate the resources used by the tests
 
-    with utils.MockedProjectCacheDir(project_key), utils.MockedClient(mock_all=False) as (client, mr):
+    with utils.MockedProjectCacheDir(), utils.MockedClient(mock_all=False) as (client, mr):
         # Prepare model and dataset
         if internal:
-            utils.local_save_model_under_giskard_home_cache(model, project_key)
+            utils.local_save_model_under_giskard_home_cache(model)
         else:
             utils.register_uri_for_model_meta_info(mr, model, project_key)
             utils.register_uri_for_model_artifact_info(mr, model, project_key, register_file_contents=True)
@@ -419,10 +419,10 @@ def test_websocket_actor_dataset_processing_empty(internal, request):
 
     project_key = str(uuid.uuid4())  # Use a UUID to separate the resources used by the tests
 
-    with utils.MockedProjectCacheDir(project_key), utils.MockedClient(mock_all=False) as (client, mr):
+    with utils.MockedProjectCacheDir(), utils.MockedClient(mock_all=False) as (client, mr):
         # Prepare dataset
         if internal:
-            utils.local_save_dataset_under_giskard_home_cache(dataset, project_key)
+            utils.local_save_dataset_under_giskard_home_cache(dataset)
         else:
             utils.register_uri_for_dataset_meta_info(mr, dataset, project_key)
             utils.register_uri_for_dataset_artifact_info(mr, dataset, project_key, register_file_contents=True)
@@ -460,9 +460,9 @@ def test_websocket_actor_dataset_processing_head_slicing_with_cache(callable_und
     project_key = str(uuid.uuid4())  # Use a UUID to separate the resources used by the tests
     callable_function_project_key = project_key if callable_under_project else None
 
-    with utils.MockedProjectCacheDir(project_key):
+    with utils.MockedProjectCacheDir():
         # Prepare dataset
-        utils.local_save_dataset_under_giskard_home_cache(dataset, project_key)
+        utils.local_save_dataset_under_giskard_home_cache(dataset)
 
         head_slice.meta.uuid = str(uuid.uuid4())
 
@@ -510,9 +510,9 @@ def test_websocket_actor_dataset_processing_do_nothing_transform_with_cache(call
 
     project_key = str(uuid.uuid4())  # Use a UUID to separate the resources used by the tests
 
-    with utils.MockedProjectCacheDir(project_key):
+    with utils.MockedProjectCacheDir():
         # Prepare dataset
-        utils.local_save_dataset_under_giskard_home_cache(dataset, project_key)
+        utils.local_save_dataset_under_giskard_home_cache(dataset)
         callable_function_project_key = project_key if callable_under_project else None
 
         do_nothing.meta.uuid = str(uuid.uuid4())
@@ -552,9 +552,9 @@ def test_websocket_actor_create_sub_dataset(request):
     dataset: Dataset = request.getfixturevalue("enron_data")
     project_key = str(uuid.uuid4())
 
-    with utils.MockedProjectCacheDir(project_key=project_key), utils.MockedClient(mock_all=False) as (client, mr):
+    with utils.MockedProjectCacheDir(), utils.MockedClient(mock_all=False) as (client, mr):
         # Prepare dataset
-        utils.local_save_dataset_under_giskard_home_cache(dataset, project_key)
+        utils.local_save_dataset_under_giskard_home_cache(dataset)
         utils.register_uri_for_dataset_meta_info(mr, dataset, project_key)
 
         # Prepare dataset upload requests