Deprecated DatasetFactory class and refactored code. (#254)

Author: lu-ohai

ads/dataset/dataset.py

@@ -31,6 +31,8 @@
     DatasetDefaults,
     deprecate_default_value,
     deprecate_variable,
+    get_dataset,
+    infer_target_type,
 )
 from ads.dataset.label_encoder import DataFrameLabelEncoder
 from ads.dataset.pipeline import TransformerPipeline
@@ -223,7 +225,8 @@ def _head(self, n=5):
 
         Examples
         --------
-        >>> ds = DatasetFactory.open("classfication_data.csv")
+        >>> import pandas as pd
+        >>> ds = ADSDataset.from_dataframe(pd.read_csv("classfication_data.csv"))
         >>> ds.head()
         * displays the first 5 rows of the dataset, just as the traditional head() function would *
         """
@@ -298,7 +301,8 @@ def call(self, func, *args, sample_size=None, **kwargs):
 
         Examples
         --------
-        >>> ds = DatasetFactory.open("classfication_data.csv")
+        >>> import pandas as pd
+        >>> ds = ADSDataset.from_dataframe(pd.read_csv("classfication_data.csv"))
         >>> def f1(df):
         ...  return(sum(df), axis=0)
         >>> sum_ds = ds.call(f1)
@@ -340,20 +344,19 @@ def set_target(self, target, type_discovery=True, target_type=None):
 
         Examples
         --------
-        >>> ds = DatasetFactory.open("classfication_data.csv")
+        >>> import pandas as pd
+        >>> ds = ADSDataset.from_dataframe(pd.read_csv("classfication_data.csv"))
         >>> ds_with_target= ds.set_target("target_class")
         """
-        from ads.dataset.factory import DatasetFactory
-
         if target_type:
             target_series = self.sampled_df[target].astype(target_type)
         else:
             target_series = self.sampled_df[target]
-        return DatasetFactory._get_dataset(
+        return get_dataset(
             self.df,
             self.sampled_df,
             target,
-            DatasetFactory.infer_target_type(target, target_series, type_discovery),
+            infer_target_type(target, target_series, type_discovery),
             self.shape,
             **self.init_kwargs,
         )
@@ -396,7 +399,8 @@ def to_pandas(self, filter=None, frac=None, include_transformer_pipeline=False):
 
         Examples
         --------
-        >>> ds = DatasetFactory.open("data.csv")
+        >>> import pandas as pd
+        >>> ds = ADSDataset.from_dataframe(pd.read_csv("data.csv"))
         >>> ds_as_df = ds.to_pandas()
 
         Notes
@@ -462,7 +466,8 @@ def to_dask(
 
         Examples
         --------
-        >>> ds = DatasetFactory.open("data.csv")
+        >>> import pandas as pd
+        >>> ds = ADSDataset.from_dataframe(pd.read_csv("data.csv"))
         >>> ds_dask = ds.to_dask()
 
         Notes
@@ -521,7 +526,8 @@ def to_h2o(self, filter=None, frac=None, include_transformer_pipeline=False):
 
         Examples
         --------
-        >>> ds = DatasetFactory.open("data.csv")
+        >>> import pandas as pd
+        >>> ds = ADSDataset.from_dataframe(pd.read_csv("data.csv"))
         >>> ds_as_h2o = ds.to_h2o()
 
         Notes
@@ -578,7 +584,8 @@ def to_xgb(self, filter=None, frac=None, include_transformer_pipeline=False):
 
         Examples
         --------
-        >>> ds = DatasetFactory.open("data.csv")
+        >>> import pandas as pd
+        >>> ds = ADSDataset.from_dataframe(pd.read_csv("data.csv"))
         >>> xgb_dmat = ds.to_xgb()
 
         Notes
@@ -617,7 +624,8 @@ def sample(self, frac=None, random_state=utils.random_state):
 
         Examples
         --------
-        >>> ds = DatasetFactory.open("data.csv")
+        >>> import pandas as pd
+        >>> ds = ADSDataset.from_dataframe(pd.read_csv("data.csv"))
         >>> ds_sample = ds.sample()
         """
         df = self.df.sample(frac=frac, random_state=random_state)
@@ -644,7 +652,8 @@ def drop_columns(self, columns):
 
         Examples
         --------
-        >>> ds = DatasetFactory.open("data.csv")
+        >>> import pandas as pd
+        >>> ds = ADSDataset.from_dataframe(pd.read_csv("data.csv"))
         >>> ds_smaller = ds.drop_columns(['col1', 'col2'])
         """
         self._validate_feature(columns)
@@ -671,7 +680,8 @@ def assign_column(self, column, arg):
 
         Examples
         --------
-        >>> ds = DatasetFactory.open("data.csv")
+        >>> import pandas as pd
+        >>> ds = ADSDataset.from_dataframe(pd.read_csv("data.csv"))
         >>> ds_same_size = ds.assign_column('target',lambda x:  x>15 if x not None)
         >>> ds_bigger = ds.assign_column('new_col', np.arange(ds.shape[0]))
         """
@@ -746,7 +756,8 @@ def rename_columns(self, columns):
 
         Examples
         --------
-        >>> ds = DatasetFactory.open("data.csv")
+        >>> import pandas as pd
+        >>> ds = ADSDataset.from_dataframe(pd.read_csv("data.csv"))
         >>> ds_renamed = ds.rename_columns({'col1': 'target'})
         """
         if isinstance(columns, list):
@@ -770,7 +781,8 @@ def set_name(self, name):
 
         Examples
         --------
-        >>> ds = DatasetFactory.open("data1.csv")
+        >>> import pandas as pd
+        >>> ds = ADSDataset.from_dataframe(pd.read_csv("data1.csv"))
         >>> ds_renamed = ds.set_name("dataset1")
         """
         self.name = name
@@ -788,7 +800,8 @@ def set_description(self, description):
 
         Examples
         --------
-        >>> ds = DatasetFactory.open("data1.csv")
+        >>> import pandas as pd
+        >>> ds = ADSDataset.from_dataframe(pd.read_csv("data1.csv"))
         >>> ds_renamed = ds.set_description("dataset1 is from "data1.csv"")
         """
         self.description = description
@@ -821,7 +834,8 @@ def snapshot(self, snapshot_dir=None, name="", storage_options=None):
 
         Examples
         --------
-        >>> ds = DatasetFactory.open("data.csv")
+        >>> import pandas as pd
+        >>> ds = ADSDataset.from_dataframe(pd.read_csv("data.csv"))
         >>> ds_uri = ds.snapshot()
         """
         if snapshot_dir is None:
@@ -873,7 +887,8 @@ def to_csv(self, path, storage_options=None, **kwargs):
 
         Examples
         --------
-        >>> ds = DatasetFactory.open("data.csv")
+        >>> import pandas as pd
+        >>> ds = ADSDataset.from_dataframe(pd.read_csv("data.csv"))
         >>> [ds_link] = ds.to_csv("my/path.csv")
         """
         if storage_options is None:
@@ -900,7 +915,8 @@ def to_parquet(self, path, storage_options=None, **kwargs):
 
         Examples
         --------
-        >>> ds = DatasetFactory.open("data.csv")
+        >>> import pandas as pd
+        >>> ds = ADSDataset.from_dataframe(pd.read_csv("data.csv"))
         >>> ds.to_parquet("my/path")
         """
         if storage_options is None:
@@ -927,7 +943,8 @@ def to_json(self, path, storage_options=None, **kwargs):
 
         Examples
         --------
-        >>> ds = DatasetFactory.open("data.csv")
+        >>> import pandas as pd
+        >>> ds = ADSDataset.from_dataframe(pd.read_csv("data.csv"))
         >>> ds.to_json("my/path.json")
         """
         if storage_options is None:
@@ -962,7 +979,8 @@ def to_hdf(
 
         Examples
         --------
-        >>> ds = DatasetFactory.open("data.csv")
+        >>> import pandas as pd
+        >>> ds = ADSDataset.from_dataframe(pd.read_csv("data.csv"))
         >>> ds.to_hdf(path="my/path.h5", key="df")
         """
         if storage_options is None:
@@ -1035,7 +1053,13 @@ def to_avro(self, path, schema=None, storage_options=None, **kwargs):
 
         Examples
         --------
-        >>> ds = DatasetFactory.open("data.avro")
+        >>> import pandas
+        >>> import fastavro
+        >>> with open("data.avro", "rb") as fp:
+        >>>     reader = fastavro.reader(fp)
+        >>>     records = [r for r in reader]
+        >>>     df = pandas.DataFrame.from_records(records)
+        >>> ds = ADSDataset.from_dataframe(df)
         >>> ds.to_avro("my/path.avro")
         """
         # Get the row by row formatting
@@ -1101,7 +1125,8 @@ def astype(self, types):
 
         Examples
         --------
-        >>> ds = DatasetFactory.open("data.csv")
+        >>> import pandas as pd
+        >>> ds = ADSDataset.from_dataframe(pd.read_csv("data.csv"))
         >>> ds_reformatted = ds.astype({"target": "categorical"})
         """
         return self.__getattr__("astype")(helper.map_types(types))
@@ -1119,8 +1144,10 @@ def merge(self, data, **kwargs):
 
         Examples
         --------
-        >>> ds1 = DatasetFactory.open("data1.csv")
-        >>> ds2 = DatasetFactory.open("data2.csv")
+        >>> import pandas as pd
+        >>> df1 = pd.read_csv("data1.csv")
+        >>> df2 = pd.read_csv("data2.csv")
+        >>> ds = ADSDataset.from_dataframe(df1.merge(df2))
         >>> ds_12 = ds1.merge(ds2)
         """
         assert isinstance(data, pd.DataFrame) or isinstance(
@@ -1275,9 +1302,8 @@ def _build_new_dataset(
             if progress:
                 progress.update("Building new dataset")
             target_type = self.target.type if target_type is None else target_type
-            from ads.dataset.factory import DatasetFactory
 
-            new_ds = DatasetFactory._get_dataset(
+            new_ds = get_dataset(
                 df,
                 sampled_df,
                 target,

ads/dataset/dataset_browser.py

@@ -19,7 +19,7 @@
 import pandas as pd
 import sklearn.datasets as sk_datasets
 
-from ads.dataset.factory import DatasetFactory
+from ads.dataset import helper
 from ads.common.utils import inject_and_copy_kwargs
 from ads.common.decorator.runtime_dependency import (
     runtime_dependency,
@@ -170,7 +170,7 @@ def open(self, name: str, **kwargs):
 
         for obj in self._generate_filelist():
             if obj["name"] == name:
-                return DatasetFactory.open(
+                return helper.open(
                     **inject_and_copy_kwargs(
                         kwargs,
                         **{
@@ -202,7 +202,7 @@ def open(self, name: str, **kwargs):
 
         for obj in self._generate_filelist():
             if obj["name"] == name:
-                return DatasetFactory.open(
+                return helper.open(
                     **inject_and_copy_kwargs(
                         kwargs,
                         **{
@@ -286,7 +286,7 @@ def open(self, name: str, **kwargs):
         #
         for obj in self.listing:
             if obj["name"] == name:
-                return DatasetFactory.open(
+                return helper.open(
                     obj["url"],
                     format=obj["format"],
                     name=obj["name"],
@@ -307,7 +307,7 @@ def list(self, filter_pattern: str = ".*") -> List[str]:
     @runtime_dependency(module="seaborn", install_from=OptionalDependency.VIZ)
     def open(self, name: str, **kwargs):
         if name in self.dataset_names:
-            return DatasetFactory.open(
+            return helper.open(
                 seaborn.load_dataset(name), name=name, description="from seaborn"
             )
         else:
@@ -350,7 +350,7 @@ def open(self, name: str, **kwargs):
                 df = pd.DataFrame(data.data, columns=data.feature_names)
                 df["target"] = pd.Series(data.target)
 
-            return DatasetFactory.open(
+            return helper.open(
                 df, target="target", name=name, description=description
             )
 

ads/dataset/factory.py

@@ -59,6 +59,17 @@
 mindate = datetime.date(datetime.MINYEAR, 1, 1)
 
 
+warnings.warn(
+    (
+        "The `ads.dataset.factory` is deprecated in `oracle-ads 2.8.8` and will be removed in `oracle-ads 3.0`."
+        "Use Pandas to read from local files or object storage directly. "
+        "Check https://accelerated-data-science.readthedocs.io/en/latest/user_guide/loading_data/connect.html."
+    ),
+    DeprecationWarning,
+    stacklevel=2,
+)
+
+
 class DatasetFactory:
     @staticmethod
     @deprecated(