Merge branch 'main' into main

Author: Davide-Cocco

.coveragerc

@@ -14,7 +14,6 @@ omit =
     ads/common/analyzer.py
     ads/common/artifact/*
     ads/common/function/*
-    ads/dataflow/*
     ads/dataset/dask_series.py
     ads/dataset/factory.py
     ads/environment/*
@@ -48,7 +47,6 @@ omit =
     ads/common/analyzer.py
     ads/common/artifact/*
     ads/common/function/*
-    ads/dataflow/*
     ads/dataset/dask_series.py
     ads/dataset/factory.py
     ads/environment/*

.github/workflows/add-3plicense-warning.yml

@@ -0,0 +1,43 @@
+name: "Add 3P License Warning to PR"
+
+on:
+  pull_request:
+    paths:
+      - pyproject.toml
+
+# Cancel in progress workflows on pull_requests.
+# https://docs.github.com/en/actions/using-jobs/using-concurrency#example-using-a-fallback-value
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+  pull-requests: write
+
+jobs:
+  warning-message-3plicense:
+    name: Add ⚠️ Warning about 3p license
+    runs-on: ubuntu-latest
+    if: ${{ success() }} && ${{ github.event.issue.pull_request }}
+    steps:
+      - run: |
+          BODY_MSG=$(cat << EOF
+          ⚠️ This PR changed **pyproject.toml** file. ⚠️
+            - PR Creator must update 📃 THIRD_PARTY_LICENSES.txt, if any 📚 library added/removed in **pyproject.toml**.
+            - PR Approver must confirm 📃 THIRD_PARTY_LICENSES.txt updated, if any 📚 library added/removed in **pyproject.toml**.
+          EOF
+          )
+          echo "BODY_MSG<<EOF" >> $GITHUB_ENV
+          echo "$BODY_MSG" >> $GITHUB_ENV
+          echo "EOF" >> $GITHUB_ENV
+      - uses: actions/github-script@v6
+        with:
+          github-token: ${{ github.token }}
+          script: |
+            github.rest.issues.createComment({
+              issue_number: context.issue.number,
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              body: `${{ env.BODY_MSG }}`
+            })

.github/workflows/codeql.yml

@@ -38,11 +38,11 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@v2
+      uses: github/codeql-action/init@v3
       with:
         languages: ${{ matrix.language }}
         # If you wish to specify custom queries, you can do so here or in a config file.
@@ -56,7 +56,7 @@ jobs:
     # Autobuild attempts to build any compiled languages  (C/C++, C#, Go, or Java).
     # If this step fails, then you should remove it and run the build manually (see below)
     - name: Autobuild
-      uses: github/codeql-action/autobuild@v2
+      uses: github/codeql-action/autobuild@v3
 
     # ℹ️ Command-line programs to run using the OS shell.
     # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
@@ -69,6 +69,6 @@ jobs:
     #   ./location_of_script_within_repo/buildscript.sh
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v2
+      uses: github/codeql-action/analyze@v3
       with:
         category: "/language:${{matrix.language}}"

.github/workflows/create-more-space/action.yml

@@ -0,0 +1,14 @@
+# We are running out space on the disk, solution described here:
+# https://github.com/actions/runner-images/issues/2840#issuecomment-790492173
+
+name: "Create more disk space"
+description: "Removing some folders to create more disk space"
+runs:
+  using: composite
+  steps:
+    - shell: bash
+      run: |
+        sudo rm -rf /usr/share/dotnet
+        sudo rm -rf /opt/ghc
+        sudo rm -rf "/usr/local/share/boost"
+        sudo rm -rf "$AGENT_TOOLSDIRECTORY"

.github/workflows/publish-to-pypi.yml

@@ -9,22 +9,22 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: "3.x"
       - name: Build distribution 📦
         run: |
-          pip install wheel
+          pip install build
           make dist
       - name: Validate
         run: |
           pip install dist/*.whl
           python -c "import ads;"
-## To run publish to test PyPI secret with token needs to be added,
+## To run publish to test PyPI a secret with token needs to be added,
 ##    this one GH_ADS_TESTPYPI_TOKEN - removed after initial test.
-## Project name also needed to be updated in setup.py - setup(name="test_oracle_ads", ...),
+## Project name also needed to be updated in pyproject.toml - name = "test_oracle_ads" in [project] section
 ##    regular name is occupied by former developer and can't be used for testing
 #      - name: Publish distribution 📦 to Test PyPI
 #        env:

.github/workflows/publish-to-readthedocs.yml

@@ -1,18 +1,11 @@
 name: "Publish Docs"
 
+# To run this workflow manually from the Actions tab
 on:
   # Auto-trigger this workflow on tag creation
   push:
     tags:
       - 'v*.*.*'
-  # Auto-trigger this workflow on merge to main changes in docs/** folder
-  pull_request_target:
-    types:
-      - closed
-    branches:
-      - main
-    paths:
-      - 'docs/**'
 
 env:
   RTDS_ADS_PROJECT: https://readthedocs.org/api/v3/projects/accelerated-data-science
@@ -24,17 +17,16 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - name: When PR ✅ merged - Trigger Readthedocs build
-        if: github.event_name == 'pull_request_target' && github.event.pull_request.merged == true
+      - name: When tag 🏷️ pushed - Trigger Readthedocs build
+        if: github.event_name == 'push' && startsWith(github.ref_name, 'v')
         run: |
+          # trigger build/publish of latest version
           curl \
             -X POST \
             -H "Authorization: Token $RTDS_ADS_TOKEN" $RTDS_ADS_PROJECT/versions/latest/builds/
-      - name: When tag 🏷️ pushed - Trigger Readthedocs build
-        if: github.event_name == 'push' && startsWith(github.ref_name, 'v')
-        run: |
-          # add 10 minutes wait time for readthedocs see freshly created tag
-          sleep 10m
+          # add 15 minutes wait time for readthedocs see freshly created tag
+          sleep 15m
+          # trigger build/publish of v*.*.* version
           curl \
             -X POST \
             -H "Authorization: Token $RTDS_ADS_TOKEN" $RTDS_ADS_PROJECT/versions/${{ github.ref_name }}/builds/

.github/workflows/run-operators-unit-tests.yml

@@ -0,0 +1,59 @@
+name: "[Py3.8][Py3.10] Operators Tests"
+
+on:
+  workflow_dispatch:
+  pull_request:
+    paths:
+      - "ads/opctl/operator/**"
+      - "**requirements.txt"
+      - ".github/workflows/run-operators*.yml"
+      - "test-requirements-operators.txt"
+
+# Cancel in progress workflows on pull_requests.
+# https://docs.github.com/en/actions/using-jobs/using-concurrency#example-using-a-fallback-value
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+# hack for https://github.com/actions/cache/issues/810#issuecomment-1222550359
+env:
+  SEGMENT_DOWNLOAD_TIMEOUT_MINS: 5
+
+jobs:
+  test:
+    name: python ${{ matrix.python-version }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 180
+
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.8"]
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: "pip"
+          cache-dependency-path: |
+            pyproject.toml
+            "**requirements.txt"
+            "test-requirements-operators.txt"
+
+      - uses: ./.github/workflows/set-dummy-conf
+        name: "Test config setup"
+
+      - name: "Run Operators Tests"
+        timeout-minutes: 180
+        shell: bash
+        run: |
+          set -x # print commands that are executed
+          $CONDA/bin/conda init
+          source /home/runner/.bashrc
+          pip install -r test-requirements-operators.txt
+          python -m pytest -v -p no:warnings --durations=5 tests/operators

.github/workflows/run-unittests-default_setup.yml

@@ -1,18 +1,15 @@
-name: tests/unitary/default_setup/**
+name: "[Py3.8][Py3.9][Py3.10] tests/unitary/default_setup/**"
 
 on:
+  workflow_dispatch:
   pull_request:
-    branches:
-      - main
-      - "release/**"
-      - develop
+    branches: [ "main" ]
     paths:
       - "ads/**"
-      - "!ads/ads_version.json"
-      - setup.py
-      - "**requirements.txt"
-      - .github/workflows/run-unittests.yml
-      - .github/workflows/run-unittests-default_setup.yml
+      - "!ads/opctl/operator/**"
+      - "!ads/feature_store/**"
+      - "pyproject.toml"
+
 
 # Cancel in progress workflows on pull_requests.
 # https://docs.github.com/en/actions/using-jobs/using-concurrency#example-using-a-fallback-value
@@ -39,48 +36,29 @@ jobs:
         python-version: ["3.8", "3.9", "3.10"]
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
-      # Caching python libraries installed with pip
-      # https://github.com/actions/cache/blob/main/examples.md#python---pip
-      - uses: actions/cache@v3
-        with:
-          path: ~/.cache/pip
-          key: ${{ runner.os }}-pip-${{ hashFiles('**/test-requirements.txt') }}
-          restore-keys: |
-            ${{ runner.os }}-pip-
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
+          cache: "pip"
+          cache-dependency-path: |
+            pyproject.toml
+            "**requirements.txt"
 
-      - name: "Test config setup"
-        shell: bash
-        env:
-          HOME_RUNNER_DIR: /home/runner
-        run: |
-          set -x # print commands that are executed
-          mkdir -p "$HOME_RUNNER_DIR"/.oci
-          openssl genrsa -out $HOME_RUNNER_DIR/.oci/oci_ads_user.pem 2048
-          cat <<EOT >> "$HOME_RUNNER_DIR/.oci/config"
-          [DEFAULT]
-          user=ocid1.user.oc1..xxx
-          fingerprint=00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00
-          tenancy=ocid1.tenancy.oc1..xxx
-          region=test_region
-          key_file=$HOME_RUNNER_DIR/.oci/oci_ads_user.pem
-          EOT
-          ls -lha "$HOME_RUNNER_DIR"/.oci
-          echo "Test config file:"
-          cat $HOME_RUNNER_DIR/.oci/config
+      - uses: ./.github/workflows/set-dummy-conf
+        name: "Test config setup"
 
       - name: "Run default_setup tests folder ONLY with minimum ADS dependencies"
-        timeout-minutes: 15
+        timeout-minutes: 30
         shell: bash
         env:
           NoDependency: True
         run: |
           set -x # print commands that are executed
           $CONDA/bin/conda init
           source /home/runner/.bashrc
+          conda install python=${{ matrix.python-version }}
           pip install -r test-requirements.txt
+          conda list
           python -m pytest -v -p no:warnings --durations=5 tests/unitary/default_setup