Feature/forecast auto select (#825)

Author: ahosler

ads/opctl/operator/lowcode/anomaly/model/base_model.py

@@ -4,33 +4,29 @@
 # Copyright (c) 2023, 2024 Oracle and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
 
+import fsspec
+import numpy as np
 import os
+import pandas as pd
 import tempfile
 import time
 from abc import ABC, abstractmethod
-from typing import Tuple
-
-import fsspec
-import pandas as pd
-import numpy as np
 from sklearn import linear_model
+from typing import Tuple
 
+from ads.common.object_storage_details import ObjectStorageDetails
 from ads.opctl import logger
-
-from ..operator_config import AnomalyOperatorConfig, AnomalyOperatorSpec
-from .anomaly_dataset import AnomalyDatasets, AnomalyOutput, TestData
 from ads.opctl.operator.lowcode.anomaly.const import OutputColumns, SupportedMetrics
-from ..const import SupportedModels
+from ads.opctl.operator.lowcode.anomaly.utils import _build_metrics_df, default_signer
 from ads.opctl.operator.lowcode.common.utils import (
     human_time_friendly,
     enable_print,
     disable_print,
     write_data,
-    merge_category_columns,
-    find_output_dirname,
 )
-from ads.opctl.operator.lowcode.anomaly.utils import _build_metrics_df, default_signer
-from ads.common.object_storage_details import ObjectStorageDetails
+from .anomaly_dataset import AnomalyDatasets, AnomalyOutput, TestData
+from ..const import SupportedModels
+from ..operator_config import AnomalyOperatorConfig, AnomalyOperatorSpec
 
 
 class AnomalyOperatorBaseModel(ABC):
@@ -246,7 +242,7 @@ def _save_report(
         """Saves resulting reports to the given folder."""
         import report_creator as rc
 
-        unique_output_dir = find_output_dirname(self.spec.output_directory)
+        unique_output_dir = self.spec.output_directory.url
 
         if ObjectStorageDetails.is_oci_path(unique_output_dir):
             storage_options = default_signer()

ads/opctl/operator/lowcode/anomaly/operator_config.py

@@ -16,6 +16,7 @@
     InputData,
 )
 from .const import SupportedModels
+from ads.opctl.operator.lowcode.common.utils import find_output_dirname
 
 
 @dataclass(repr=True)
@@ -79,6 +80,7 @@ class AnomalyOperatorSpec(DataClassSerializable):
 
     def __post_init__(self):
         """Adjusts the specification details."""
+        self.output_directory = self.output_directory or OutputDirectory(url=find_output_dirname(self.output_directory))
         self.report_file_name = self.report_file_name or "report.html"
         self.report_theme = self.report_theme or "light"
         self.inliers_filename = self.inliers_filename or "inliers.csv"

ads/opctl/operator/lowcode/common/transformations.py

@@ -97,7 +97,8 @@ def _set_series_id_column(self, df):
                 for value in merged_values:
                     self._target_category_columns_map[value] = df[df[DataColumns.Series] == value][self.target_category_columns].drop_duplicates().iloc[0].to_dict()
 
-            df = df.drop(self.target_category_columns, axis=1)
+            if self.target_category_columns != [DataColumns.Series]:
+                df = df.drop(self.target_category_columns, axis=1)
         return df
 
     def _format_datetime_col(self, df):

ads/opctl/operator/lowcode/common/utils.py

@@ -215,7 +215,7 @@ def human_time_friendly(seconds):
 
 
 def find_output_dirname(output_dir: OutputDirectory):
-    if output_dir:
+    if output_dir and output_dir.url:
         return output_dir.url
     output_dir = "results"
 

ads/opctl/operator/lowcode/forecast/__main__.py

@@ -24,24 +24,9 @@ def operate(operator_config: ForecastOperatorConfig) -> None:
     from .model.factory import ForecastOperatorModelFactory
 
     datasets = ForecastDatasets(operator_config)
-    try:
-        ForecastOperatorModelFactory.get_model(
-            operator_config, datasets
-        ).generate_report()
-    except Exception as e:
-        if operator_config.spec.model == "auto":
-            logger.debug(
-                f"Failed to forecast with error {e.args}. Trying again with model `prophet`."
-            )
-            operator_config.spec.model = "prophet"
-            operator_config.spec.model_kwargs = dict()
-            datasets = ForecastDatasets(operator_config)
-            ForecastOperatorModelFactory.get_model(
-                operator_config, datasets
-            ).generate_report()
-        else:
-            raise
-
+    ForecastOperatorModelFactory.get_model(
+        operator_config, datasets
+    ).generate_report()
 
 def verify(spec: Dict, **kwargs: Dict) -> bool:
     """Verifies the forecasting operator config."""

ads/opctl/operator/lowcode/forecast/const.py

@@ -87,3 +87,4 @@ class ForecastOutputColumns(str, metaclass=ExtendedEnumMeta):
 SUMMARY_METRICS_HORIZON_LIMIT = 10
 PROPHET_INTERNAL_DATE_COL = "ds"
 RENDER_LIMIT = 5000
+AUTO_SELECT = "auto-select"

ads/opctl/operator/lowcode/forecast/model/base_model.py

@@ -4,31 +4,19 @@
 # Copyright (c) 2023, 2024 Oracle and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
 
-import json
+import fsspec
+import numpy as np
 import os
+import pandas as pd
 import tempfile
 import time
+import traceback
 from abc import ABC, abstractmethod
 from typing import Tuple
-import traceback
-
-import fsspec
-import numpy as np
-import pandas as pd
 
-from ads.opctl.operator.lowcode.forecast.utils import (
-    default_signer,
-    evaluate_train_metrics,
-    get_forecast_plots,
-    _build_metrics_df,
-    _build_metrics_per_horizon,
-    load_pkl,
-    write_pkl,
-    _label_encode_dataframe,
-)
+from ads.common.decorator.runtime_dependency import runtime_dependency
 from ads.common.object_storage_details import ObjectStorageDetails
 from ads.opctl import logger
-
 from ads.opctl.operator.lowcode.common.utils import (
     human_time_friendly,
     enable_print,
@@ -37,18 +25,28 @@
     merged_category_column_name,
     datetime_to_seconds,
     seconds_to_datetime,
-    find_output_dirname,
 )
+from ads.opctl.operator.lowcode.forecast.model.forecast_datasets import TestData
+from ads.opctl.operator.lowcode.forecast.utils import (
+    default_signer,
+    evaluate_train_metrics,
+    get_forecast_plots,
+    get_auto_select_plot,
+    _build_metrics_df,
+    _build_metrics_per_horizon,
+    load_pkl,
+    write_pkl,
+    _label_encode_dataframe,
+)
+from .forecast_datasets import ForecastDatasets
 from ..const import (
     SUMMARY_METRICS_HORIZON_LIMIT,
     SupportedMetrics,
     SupportedModels,
     SpeedAccuracyMode,
+    AUTO_SELECT
 )
 from ..operator_config import ForecastOperatorConfig, ForecastOperatorSpec
-from ads.common.decorator.runtime_dependency import runtime_dependency
-from .forecast_datasets import ForecastDatasets, ForecastOutput
-from ads.opctl.operator.lowcode.forecast.model.forecast_datasets import TestData
 
 
 class ForecastOperatorBaseModel(ABC):
@@ -250,6 +248,23 @@ def generate_report(self):
                     sec9 = rc.DataTable(self.eval_metrics, index=True)
                     train_metrics_sections = [sec9_text, sec9]
 
+                backtest_sections = []
+                if self.spec.model == AUTO_SELECT:
+                    output_dir = self.spec.output_directory.url
+                    backtest_report_name = "backtest_stats.csv"
+                    backtest_stats = pd.read_csv(f"{output_dir}/{backtest_report_name}")
+                    average_dict = backtest_stats.mean().to_dict()
+                    del average_dict['backtest']
+                    best_model = min(average_dict, key=average_dict.get)
+                    backtest_text = rc.Heading("Back Testing Metrics", level=2)
+                    summary_text = rc.Text(
+                        f"Overall, the average scores for the models are {average_dict}, with {best_model}"
+                        f" being identified as the top-performing model during backtesting.")
+                    backtest_table = rc.DataTable(backtest_stats, index=True)
+                    liner_plot = get_auto_select_plot(backtest_stats)
+                    backtest_sections = [backtest_text, backtest_table, summary_text, liner_plot]
+
+
                 forecast_plots = []
                 if len(self.forecast_output.list_series_ids()) > 0:
                     forecast_text = rc.Heading(
@@ -276,6 +291,7 @@ def generate_report(self):
                 yaml_appendix = rc.Yaml(self.config.to_dict())
                 report_sections = (
                     [summary]
+                    + backtest_sections
                     + forecast_plots
                     + other_sections
                     + test_metrics_sections
@@ -409,7 +425,7 @@ def _save_report(
         """Saves resulting reports to the given folder."""
         import report_creator as rc
 
-        unique_output_dir = find_output_dirname(self.spec.output_directory)
+        unique_output_dir = self.spec.output_directory.url
 
         if ObjectStorageDetails.is_oci_path(unique_output_dir):
             storage_options = default_signer()

ads/opctl/operator/lowcode/forecast/model/factory.py

@@ -4,17 +4,17 @@
 # Copyright (c) 2023 Oracle and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
 
-from ..const import SupportedModels
+from ..const import SupportedModels, AUTO_SELECT
 from ..operator_config import ForecastOperatorConfig
 from .arima import ArimaOperatorModel
 from .automlx import AutoMLXOperatorModel
 from .autots import AutoTSOperatorModel
 from .base_model import ForecastOperatorBaseModel
 from .neuralprophet import NeuralProphetOperatorModel
 from .prophet import ProphetOperatorModel
-from .ml_forecast import MLForecastOperatorModel
-from ..utils import select_auto_model
 from .forecast_datasets import ForecastDatasets
+from .ml_forecast import MLForecastOperatorModel
+from ..model_evaluator import ModelEvaluator
 
 class UnSupportedModelError(Exception):
     def __init__(self, model_type: str):
@@ -63,8 +63,35 @@ def get_model(
             In case of not supported model.
         """
         model_type = operator_config.spec.model
-        if model_type == "auto":
-            model_type = select_auto_model(datasets, operator_config)
+        if model_type == AUTO_SELECT:
+            model_type = cls.auto_select_model(datasets, operator_config)
+            operator_config.spec.model_kwargs = dict()
         if model_type not in cls._MAP:
             raise UnSupportedModelError(model_type)
         return cls._MAP[model_type](config=operator_config, datasets=datasets)
+
+    @classmethod
+    def auto_select_model(
+            cls, datasets: ForecastDatasets, operator_config: ForecastOperatorConfig
+    ) -> str:
+        """
+        Selects AutoMLX or Arima model based on column count.
+
+        If the number of columns is less than or equal to the maximum allowed for AutoMLX,
+        returns 'AutoMLX'. Otherwise, returns 'Arima'.
+
+        Parameters
+        ------------
+        datasets:  ForecastDatasets
+                Datasets for predictions
+
+        Returns
+        --------
+        str
+            The type of the model.
+        """
+        all_models = operator_config.spec.model_kwargs.get("model_list", cls._MAP.keys())
+        num_backtests = operator_config.spec.model_kwargs.get("num_backtests", 5)
+        sample_ratio = operator_config.spec.model_kwargs.get("sample_ratio", 0.20)
+        model_evaluator = ModelEvaluator(all_models, num_backtests, sample_ratio)
+        return model_evaluator.find_best_model(datasets, operator_config)