Merge branch 'main' into release/v2.13.0

Author: mayoor

ads/model/artifact_downloader.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python
-# -*- coding: utf-8; -*-
 
 # Copyright (c) 2022, 2024 Oracle and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
@@ -12,9 +11,9 @@
 from zipfile import ZipFile
 
 from ads.common import utils
+from ads.common.object_storage_details import ObjectStorageDetails
 from ads.common.utils import extract_region
 from ads.model.service.oci_datascience_model import OCIDataScienceModel
-from ads.common.object_storage_details import ObjectStorageDetails
 
 
 class ArtifactDownloader(ABC):
@@ -169,9 +168,9 @@ def __init__(
 
     def _download(self):
         """Downloads model artifacts."""
-        self.progress.update(f"Importing model artifacts from catalog")
+        self.progress.update("Importing model artifacts from catalog")
 
-        if self.dsc_model.is_model_by_reference() and self.model_file_description:
+        if self.dsc_model._is_model_by_reference() and self.model_file_description:
             self.download_from_model_file_description()
             self.progress.update()
             return

ads/model/datascience_model.py

@@ -1778,7 +1778,7 @@ def _update_from_oci_dsc_model(
             artifact_info = self.dsc_model.get_artifact_info()
             _, file_name_info = cgi.parse_header(artifact_info["Content-Disposition"])
 
-            if self.dsc_model.is_model_by_reference():
+            if self.dsc_model._is_model_by_reference():
                 _, file_extension = os.path.splitext(file_name_info["filename"])
                 if file_extension.lower() == ".json":
                     bucket_uri, _ = self._download_file_description_artifact()

ads/model/model_metadata.py

@@ -1509,7 +1509,10 @@ def _from_oci_metadata(cls, metadata_list):
         metadata = cls()
         for oci_item in metadata_list:
             item = ModelTaxonomyMetadataItem._from_oci_metadata(oci_item)
-            metadata[item.key].update(value=item.value)
+            if item.key in metadata.keys:
+                metadata[item.key].update(value=item.value)
+            else:
+                metadata._items.add(item)
         return metadata
 
     def to_dataframe(self) -> pd.DataFrame:
@@ -1562,7 +1565,10 @@ def from_dict(cls, data: Dict) -> "ModelTaxonomyMetadata":
         metadata = cls()
         for item in data["data"]:
             item = ModelTaxonomyMetadataItem.from_dict(item)
-            metadata[item.key].update(value=item.value)
+            if item.key in metadata.keys:
+                metadata[item.key].update(value=item.value)
+            else:
+                metadata._items.add(item)
         return metadata
 
 

ads/model/service/oci_datascience_model.py

@@ -1,35 +1,32 @@
 #!/usr/bin/env python
-# -*- coding: utf-8; -*-
 
 # Copyright (c) 2022, 2024 Oracle and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
 
 import logging
-import time
 from functools import wraps
 from io import BytesIO
 from typing import Callable, Dict, List, Optional
 
 import oci.data_science
-from ads.common import utils
-from ads.common.object_storage_details import ObjectStorageDetails
-from ads.common.oci_datascience import OCIDataScienceMixin
-from ads.common.oci_mixin import OCIWorkRequestMixin
-from ads.common.oci_resource import SEARCH_TYPE, OCIResource
-from ads.common.utils import extract_region
-from ads.common.work_request import DataScienceWorkRequest
-from ads.model.deployment import ModelDeployment
 from oci.data_science.models import (
     ArtifactExportDetailsObjectStorage,
     ArtifactImportDetailsObjectStorage,
     CreateModelDetails,
     ExportModelArtifactDetails,
     ImportModelArtifactDetails,
     UpdateModelDetails,
-    WorkRequest,
 )
 from oci.exceptions import ServiceError
 
+from ads.common.object_storage_details import ObjectStorageDetails
+from ads.common.oci_datascience import OCIDataScienceMixin
+from ads.common.oci_mixin import OCIWorkRequestMixin
+from ads.common.oci_resource import SEARCH_TYPE, OCIResource
+from ads.common.utils import extract_region
+from ads.common.work_request import DataScienceWorkRequest
+from ads.model.deployment import ModelDeployment
+
 logger = logging.getLogger(__name__)
 
 _REQUEST_INTERVAL_IN_SEC = 3
@@ -282,7 +279,7 @@ def get_artifact_info(self) -> Dict:
         msg="Model needs to be restored before the archived artifact content can be accessed."
     )
     def restore_archived_model_artifact(
-            self, restore_model_for_hours_specified: Optional[int] = None
+        self, restore_model_for_hours_specified: Optional[int] = None
     ) -> None:
         """Restores the archived model artifact.
 
@@ -304,7 +301,8 @@ def restore_archived_model_artifact(
         """
         return self.client.restore_archived_model_artifact(
             model_id=self.id,
-            restore_model_for_hours_specified=restore_model_for_hours_specified).headers["opc-work-request-id"]
+            restore_model_for_hours_specified=restore_model_for_hours_specified,
+        ).headers["opc-work-request-id"]
 
     @check_for_model_id(
         msg="Model needs to be saved to the Model Catalog before the artifact content can be read."
@@ -581,7 +579,7 @@ def from_id(cls, ocid: str) -> "OCIDataScienceModel":
             raise ValueError("Model OCID not provided.")
         return super().from_ocid(ocid)
 
-    def is_model_by_reference(self):
+    def _is_model_by_reference(self):
         """Checks if model is created by reference
         Returns
         -------

tests/unitary/default_setup/model/test_artifact_downloader.py

@@ -189,7 +189,7 @@ def test_download_large_artifact_from_model_file_description(self, mock_download
         """Tests whether model_file_description is loaded within downloader and is parsed, and also if
         #  download_from_model_file_description is appropriately called."""
 
-        self.mock_dsc_model.is_model_by_reference.return_value = True
+        self.mock_dsc_model._is_model_by_reference.return_value = True
         self.mock_artifact_file_path = os.path.join(
             self.curr_dir, "test_files/model_description.json"
         )

tests/unitary/default_setup/model/test_datascience_model.py

@@ -26,7 +26,9 @@
     ModelArtifactSizeError,
     BucketNotVersionedError,
     ModelFileDescriptionError,
-    InvalidArtifactType, ModelRetentionSetting, ModelBackupSetting,
+    InvalidArtifactType,
+    ModelRetentionSetting,
+    ModelBackupSetting,
 )
 from ads.model.model_metadata import (
     ModelCustomMetadata,
@@ -44,7 +46,7 @@
 from ads.config import AQUA_SERVICE_MODELS_BUCKET as SERVICE_MODELS_BUCKET
 
 MODEL_OCID = "ocid1.datasciencemodel.oc1.iad.<unique_ocid>"
- 
+
 OCI_MODEL_PAYLOAD = {
     "id": MODEL_OCID,
     "compartment_id": "ocid1.compartment.oc1..<unique_ocid>",
@@ -71,16 +73,17 @@
         {"key": "UseCaseType", "value": "multinomial_classification"},
         {"key": "Hyperparameters"},
         {"key": "ArtifactTestResults"},
+        {"key": "UnexpectedKey", "value": "unexpected_value"},
     ],
     "backup_setting": {
         "is_backup_enabled": True,
         "backup_region": "us-phoenix-1",
-        "customer_notification_type": "ALL"
+        "customer_notification_type": "ALL",
     },
     "retention_setting": {
         "archive_after_days": 30,
         "delete_after_days": 90,
-        "customer_notification_type": "ALL"
+        "customer_notification_type": "ALL",
     },
     "input_schema": '{"schema": [{"dtype": "int64", "feature_type": "Integer", "name": 0, "domain": {"values": "", "stats": {}, "constraints": []}, "required": true, "description": "0", "order": 0}], "version": "1.1"}',
     "output_schema": '{"schema": [{"dtype": "int64", "feature_type": "Integer", "name": 0, "domain": {"values": "", "stats": {}, "constraints": []}, "required": true, "description": "0", "order": 0}], "version": "1.1"}',
@@ -148,6 +151,7 @@
             {"key": "UseCaseType", "value": "multinomial_classification"},
             {"key": "Hyperparameters", "value": None},
             {"key": "ArtifactTestResults", "value": None},
+            {"key": "UnexpectedKey", "value": "unexpected_value"},
         ]
     },
     "provenanceMetadata": {
@@ -161,12 +165,12 @@
     "backupSetting": {
         "is_backup_enabled": True,
         "backup_region": "us-phoenix-1",
-        "customer_notification_type": "ALL"
+        "customer_notification_type": "ALL",
     },
     "retentionSetting": {
         "archive_after_days": 30,
         "delete_after_days": 90,
-        "customer_notification_type": "ALL"
+        "customer_notification_type": "ALL",
     },
     "artifact": "ocid1.datasciencemodel.oc1.iad.<unique_ocid>.zip",
 }
@@ -327,8 +331,8 @@ def test_with_methods_1(self, mock_load_default_properties):
             .with_defined_metadata_list(self.payload["definedMetadataList"])
             .with_provenance_metadata(self.payload["provenanceMetadata"])
             .with_artifact(self.payload["artifact"])
-            .with_backup_setting(self.payload['backupSetting'])
-            .with_retention_setting(self.payload['retentionSetting'])
+            .with_backup_setting(self.payload["backupSetting"])
+            .with_retention_setting(self.payload["retentionSetting"])
         )
         assert self.prepare_dict(dsc_model.to_dict()["spec"]) == self.prepare_dict(
             self.payload
@@ -356,8 +360,12 @@ def test_with_methods_2(self):
                 ModelProvenanceMetadata.from_dict(self.payload["provenanceMetadata"])
             )
             .with_artifact(self.payload["artifact"])
-            .with_backup_setting(ModelBackupSetting.from_dict(self.payload['backupSetting']))
-            .with_retention_setting(ModelRetentionSetting.from_dict(self.payload['retentionSetting']))
+            .with_backup_setting(
+                ModelBackupSetting.from_dict(self.payload["backupSetting"])
+            )
+            .with_retention_setting(
+                ModelRetentionSetting.from_dict(self.payload["retentionSetting"])
+            )
         )
         assert self.prepare_dict(dsc_model.to_dict()["spec"]) == self.prepare_dict(
             self.payload
@@ -605,7 +613,7 @@ def test__to_oci_dsc_model(self):
             True,
         ],
     )
-    @patch.object(OCIDataScienceModel, "is_model_by_reference")
+    @patch.object(OCIDataScienceModel, "_is_model_by_reference")
     @patch.object(OCIDataScienceModel, "get_artifact_info")
     @patch.object(OCIDataScienceModel, "get_model_provenance")
     @patch.object(DataScienceModel, "_download_file_description_artifact")

tests/unitary/default_setup/model/test_model_artifact.py

@@ -7,7 +7,7 @@
 import pickle
 import sys
 
-import mock
+from unittest import mock
 import pytest
 import yaml
 from ads.common.model import ADSModel