Release/2.8.x (#115)

Author: Ziqun Ye

ads/__init__.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*--
 
-# Copyright (c) 2020, 2022 Oracle and/or its affiliates.
+# Copyright (c) 2020, 2023 Oracle and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
 
 from __future__ import print_function, division, absolute_import
@@ -26,6 +26,7 @@
 from ads.feature_engineering.accessor.dataframe_accessor import ADSDataFrameAccessor
 from ads.common import auth
 from ads.common.auth import set_auth
+from ads.common.config import Config
 
 os.environ["GIT_PYTHON_REFRESH"] = "quiet"
 

ads/ads_version.json

@@ -1,3 +1,3 @@
 {
-  "version": "2.8.2"
+  "version": "2.8.3"
 }

ads/common/config.py

@@ -1,18 +1,20 @@
 #!/usr/bin/env python
 # -*- coding: utf-8; -*-
 
-# Copyright (c) 2022 Oracle and/or its affiliates.
+# Copyright (c) 2022, 2023 Oracle and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
 
 import os
 from collections import defaultdict
 from configparser import ConfigParser
 from copy import copy
 from enum import Enum
-from typing import Callable, Dict, List, Optional, Tuple, Union, Any
+from typing import Any, Callable, Dict, List, Optional, Tuple, Union
+from urllib.parse import urlparse
 
 import fsspec
 import yaml
+
 from ads.common import auth as authutil
 from ads.common.decorator.argument_to_case import ArgumentCase, argument_to_case
 
@@ -287,6 +289,7 @@ def __init__(
         self._config_parser = ExtendedConfigParser(uri=self.uri, auth=self.auth)
 
     def _on_change(self):
+        """This method will be called when config modified."""
         pass
 
     def default(self) -> ConfigSection:
@@ -345,7 +348,7 @@ def section_set(
         The new config section will be added in case if it doesn't exist.
         Otherwise the existing config section will be merged with the new fields.
 
-        Paramaters
+        Parameters
         ----------
         key: str
             A key of a config section.
@@ -362,7 +365,7 @@ def section_set(
         Raises
         ------
         ValueError
-            If section with goven key is already exist and `replace` flag set to False.
+            If section with given key is already exist and `replace` flag set to False.
         TypeError
             If input `info` has a wrong format.
         """
@@ -389,7 +392,7 @@ def section_set(
         return self._config[key]
 
     @argument_to_case(case=ArgumentCase.UPPER, arguments=["key"])
-    def section_remove(self, key: str) -> None:
+    def section_remove(self, key: str) -> "Config":
         """Removes config section form config.
 
         Parameters
@@ -404,26 +407,62 @@ def section_remove(self, key: str) -> None:
         """
         self._config.pop(key, None)
         self._on_change()
+        return self
+
+    def save(
+        self,
+        uri: Optional[str] = None,
+        auth: Optional[Dict] = None,
+        force_overwrite: Optional[bool] = False,
+    ) -> "Config":
+        """Saves config to a config file.
 
-    def save(self) -> None:
-        """Saves config data to a config file.
+        Parameters
+        ----------
+        uri: (str, optional). Defaults to `~/.ads/config`.
+            The path to the config file. Can be local or Object Storage file.
+        auth: (Dict, optional). Defaults to None.
+            The default authentication is set using `ads.set_auth` API. If you need to override the
+            default, use the `ads.common.auth.api_keys` or `ads.common.auth.resource_principal` to create appropriate
+            authentication signer and kwargs required to instantiate IdentityClient object.
+        force_overwrite: (bool, optional). Defaults to `False`.
+            Overwrites the config if exists.
 
         Returns
         -------
         None
             Nothing
         """
-        self._config_parser.with_dict(self.to_dict()).save()
+        uri = uri or self.uri
+        auth = auth or self.auth or authutil.default_signer()
+        self._config_parser.with_dict(self.to_dict()).save(
+            uri=uri, auth=auth, force_overwrite=force_overwrite
+        )
+        return self
+
+    def load(self, uri: Optional[str] = None, auth: Optional[Dict] = None) -> "Config":
+        """Loads config from a config file.
 
-    def load(self) -> "Config":
-        """Loads config data from a config file.
+        Parameters
+        ----------
+        uri: (str, optional). Defaults to `~/.ads/config`.
+            The path where the config file needs to be saved. Can be local or Object Storage file.
+        auth: (Dict, optional). Defaults to None.
+            The default authentication is set using `ads.set_auth` API. If you need to override the
+            default, use the `ads.common.auth.api_keys` or `ads.common.auth.resource_principal` to create appropriate
+            authentication signer and kwargs required to instantiate IdentityClient object.
 
         Returns
         -------
         Config
             A config object.
         """
-        return self.with_dict(self._config_parser.read().to_dict())
+        uri = uri or self.uri
+        auth = auth or self.auth or authutil.default_signer()
+
+        return self.with_dict(
+            self._config_parser.read(uri=uri, auth=auth).to_dict(), replace=True
+        )
 
     def with_dict(
         self,
@@ -507,23 +546,54 @@ def __init__(
         uri: (str, optional). Defaults to `~/.ads/config`.
             The path to the config file. Can be local or Object Storage file.
         auth: (Dict, optional). Defaults to None.
-            The default authetication is set using `ads.set_auth` API. If you need to override the
+            The default authentication is set using `ads.set_auth` API. If you need to override the
             default, use the `ads.common.auth.api_keys` or `ads.common.auth.resource_principal` to create appropriate
             authentication signer and kwargs required to instantiate IdentityClient object.
         """
         super().__init__(default_section="EXCLUDE_DEFAULT_SECTION")
         self.auth = auth or authutil.default_signer()
         self.uri = uri
 
-    def save(self) -> None:
+    def save(
+        self,
+        uri: Optional[str] = None,
+        auth: Optional[Dict] = None,
+        force_overwrite: Optional[bool] = False,
+    ) -> None:
         """Saves the config to the file.
 
+        Parameters
+        ----------
+        uri: (str, optional). Defaults to `~/.ads/config`.
+            The path to the config file. Can be local or Object Storage file.
+        auth: (Dict, optional). Defaults to None.
+            The default authentication is set using `ads.set_auth` API. If you need to override the
+            default, use the `ads.common.auth.api_keys` or `ads.common.auth.resource_principal` to create appropriate
+            authentication signer and kwargs required to instantiate IdentityClient object.
+        force_overwrite: (bool, optional). Defaults to `False`.
+            Overwrites the config if exists.
+
         Returns
         -------
         None
-            nothing
+
+        Raise
+        -----
+        FileExistsError
+            In case if file exists and force_overwrite is false.
         """
-        with fsspec.open(self.uri, mode="w", **self.auth) as f:
+        uri = uri or self.uri
+        auth = auth or self.auth or authutil.default_signer()
+
+        if not force_overwrite:
+            dst_path_scheme = urlparse(uri).scheme or "file"
+            if fsspec.filesystem(dst_path_scheme, **auth).exists(uri):
+                raise FileExistsError(
+                    f"The `{uri}` exists. Set `force_overwrite` to True "
+                    "if you wish to overwrite."
+                )
+
+        with fsspec.open(uri, mode="w", **auth) as f:
             self.write(f)
 
     def to_dict(self) -> Dict[str, Any]:
@@ -532,19 +602,31 @@ def to_dict(self) -> Dict[str, Any]:
         Returns
         -------
         Dict[str, Any]
-            Cofig in a dictionary format.
+            Config in a dictionary format.
         """
         return {s: dict(self[s]) for s in self.keys() if self[s]}
 
-    def read(self) -> "ExtendedConfigParser":
+    def read(
+        self, uri: Optional[str] = None, auth: Optional[Dict] = None
+    ) -> "ExtendedConfigParser":
         """Reads config file.
 
+        uri: (str, optional). Defaults to `~/.ads/config`.
+            The path to the config file. Can be local or Object Storage file.
+        auth: (Dict, optional). Defaults to None.
+            The default authentication is set using `ads.set_auth` API. If you need to override the
+            default, use the `ads.common.auth.api_keys` or `ads.common.auth.resource_principal` to create appropriate
+            authentication signer and kwargs required to instantiate IdentityClient object.
+
         Returns
         -------
         ExtendedConfigParser
            Config parser object.
         """
-        with fsspec.open(self.uri, "r", **self.auth) as f:
+        uri = uri or self.uri
+        auth = auth or self.auth or authutil.default_signer()
+
+        with fsspec.open(uri, "r", **auth) as f:
             self.read_string(f.read())
         return self
 

ads/common/oci_datascience.py

@@ -1,15 +1,16 @@
 #!/usr/bin/env python
 # -*- coding: utf-8; -*-
 
-# Copyright (c) 2021, 2022 Oracle and/or its affiliates.
+# Copyright (c) 2021, 2023 Oracle and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
 
 import oci.data_science
 from ads.common.oci_mixin import OCIModelMixin
+from ads.common.decorator.utils import class_or_instance_method
 
 
 class OCIDataScienceMixin(OCIModelMixin):
-    @classmethod
+    @class_or_instance_method
     def init_client(cls, **kwargs) -> oci.data_science.DataScienceClient:
         return cls._init_client(client=oci.data_science.DataScienceClient, **kwargs)
 

ads/common/oci_logging.py

@@ -11,6 +11,7 @@
 
 import oci.logging
 import oci.loggingsearch
+import oci.exceptions
 from ads.common.decorator.utils import class_or_instance_method
 from ads.common.oci_mixin import OCIModelMixin, OCIWorkRequestMixin
 from ads.common.oci_resource import OCIResource, ResourceNotFoundError
@@ -370,12 +371,21 @@ def _search_logs(
             time_end=self.format_datetime(time_end),
             search_query=search_query,
         )
-        response = self.search_client.search_logs(
-            search_details, limit=LIMIT_PER_REQUEST
-        )
-        records = response.data.results
-        logger.debug("%d logs received.", len(records))
-        if len(records) >= LIMIT_PER_REQUEST:
+        result_too_large = False
+        try:
+            response = self.search_client.search_logs(
+                search_details, limit=LIMIT_PER_REQUEST
+            )
+            records = response.data.results
+            logger.debug("%d logs received.", len(records))
+        except oci.exceptions.ServiceError as ex:
+            if ex.status == 400 and "search result is too large" in ex.message:
+                logger.debug(ex.message)
+                records = []
+                result_too_large = True
+            else:
+                raise oci.exceptions.ServiceError from ex
+        if result_too_large or len(records) >= LIMIT_PER_REQUEST:
             mid = time_start + (time_end - time_start) / 2
             # The log search API used RFC3339 time format.
             # The minimum time unit of RFC3339 format is 1000 microseconds.

ads/common/utils.py

@@ -1259,12 +1259,13 @@ def copy_from_uri(
             to_path = temp_dir
         else:
             unpack_path = None
+
         fs = fsspec.filesystem(scheme, **auth)
-        try:
-            fs.get(uri, to_path, recursive=True)
-        except IsADirectoryError:
+
+        if not (uri.endswith("/") or fs.isdir(uri)) and os.path.isdir(to_path):
             to_path = os.path.join(to_path, os.path.basename(str(uri).rstrip("/")))
-            fs.get(uri, to_path, recursive=True)
+
+        fs.get(uri, to_path, recursive=True)
 
         if unpack_path:
             shutil.unpack_archive(to_path, unpack_path)