Use sanitize_array consistently

Author: jbrockmendel

doc/source/whatsnew/v3.0.0.rst

@@ -674,7 +674,7 @@ Other API changes
 - :class:`Series` "flex" methods like :meth:`Series.add` no longer allow passing a :class:`DataFrame` for ``other``; use the DataFrame reversed method instead (:issue:`46179`)
 - :meth:`CategoricalIndex.append` no longer attempts to cast different-dtype indexes to the caller's dtype (:issue:`41626`)
 - :meth:`ExtensionDtype.construct_array_type` is now a regular method instead of a ``classmethod`` (:issue:`58663`)
-- Arithmetic operations between a :class:`Series`, :class:`Index`, or :class:`ExtensionArray` with a ``list`` now consistently wrap that list with ``np.array``. To do any other kind of type inference or casting, do so explicitly before operating (:issue:`62552`)
+- Arithmetic operations between a :class:`Series`, :class:`Index`, or :class:`ExtensionArray` with a ``list`` now consistently wrap that list with an array equivalent to ``Series(my_list).array``. To do any other kind of type inference or casting, do so explicitly before operating (:issue:`62552`)
 - Comparison operations between :class:`Index` and :class:`Series` now consistently return :class:`Series` regardless of which object is on the left or right (:issue:`36759`)
 - Numpy functions like ``np.isinf`` that return a bool dtype when called on a :class:`Index` object now return a bool-dtype :class:`Index` instead of ``np.ndarray`` (:issue:`52676`)
 

pandas/core/indexes/base.py

@@ -7179,19 +7179,7 @@ def _cmp_method(self, other, op):
         else:
             other = np.asarray(other)
 
-        if is_object_dtype(self.dtype) and isinstance(other, ExtensionArray):
-            # e.g. PeriodArray, Categorical
-            result = op(self._values, other)
-
-        elif isinstance(self._values, ExtensionArray):
-            result = op(self._values, other)
-
-        elif is_object_dtype(self.dtype) and not isinstance(self, ABCMultiIndex):
-            # don't pass MultiIndex
-            result = ops.comp_method_OBJECT_ARRAY(op, self._values, other)
-
-        else:
-            result = ops.comparison_op(self._values, other, op)
+        result = ops.comparison_op(self._values, other, op)
 
         return result
 

pandas/core/ops/array_ops.py

@@ -53,7 +53,10 @@
 
 from pandas.core import roperator
 from pandas.core.computation import expressions
-from pandas.core.construction import ensure_wrapped_if_datetimelike
+from pandas.core.construction import (
+    ensure_wrapped_if_datetimelike,
+    sanitize_array,
+)
 from pandas.core.ops import missing
 from pandas.core.ops.dispatch import should_extension_dispatch
 from pandas.core.ops.invalid import invalid_comparison
@@ -263,7 +266,7 @@ def arithmetic_op(left: ArrayLike, right: Any, op):
     # casts integer dtypes to timedelta64 when operating with timedelta64 - GH#22390)
     if isinstance(right, list):
         # GH#62423
-        right = np.array(right)
+        right = sanitize_array(right, None)
     right = ensure_wrapped_if_datetimelike(right)
 
     if (
@@ -314,8 +317,8 @@ def comparison_op(left: ArrayLike, right: Any, op) -> ArrayLike:
     if isinstance(rvalues, list):
         # We don't catch tuple here bc we may be comparing e.g. MultiIndex
         #  to a tuple that represents a single entry, see test_compare_tuple_strs
-        rvalues = np.asarray(rvalues)
-        rvalues = ensure_wrapped_if_datetimelike(rvalues)
+        rvalues = sanitize_array(rvalues, None)
+    rvalues = ensure_wrapped_if_datetimelike(rvalues)
 
     if isinstance(rvalues, (np.ndarray, ABCExtensionArray)):
         # TODO: make this treatment consistent across ops and classes.

pandas/core/ops/common.py

@@ -7,8 +7,6 @@
 from functools import wraps
 from typing import TYPE_CHECKING
 
-import numpy as np
-
 from pandas._libs.lib import item_from_zerodim
 from pandas._libs.missing import is_matching_na
 
@@ -18,6 +16,11 @@
     ABCSeries,
 )
 
+from pandas.core.construction import (
+    ensure_wrapped_if_datetimelike,
+    sanitize_array,
+)
+
 if TYPE_CHECKING:
     from collections.abc import Callable
 
@@ -59,6 +62,7 @@ def _unpack_zerodim_and_defer(method: F, name: str) -> F:
     -------
     method
     """
+    is_logical = name.strip("_") in ["or", "xor", "and", "ror", "rxor", "rand"]
 
     @wraps(method)
     def new_method(self, other):
@@ -69,15 +73,14 @@ def new_method(self, other):
                 return NotImplemented
 
         other = item_from_zerodim(other)
-
-        if isinstance(self, ABCExtensionArray):
-            if isinstance(other, list):
-                # See GH#62423
-                other = np.array(other)
-                if other.dtype.kind in "mM":
-                    from pandas.core.construction import ensure_wrapped_if_datetimelike
-
-                    other = ensure_wrapped_if_datetimelike(other)
+        if (
+            isinstance(self, ABCExtensionArray)
+            and isinstance(other, list)
+            and not is_logical
+        ):
+            # See GH#62423
+            other = sanitize_array(other, None)
+            other = ensure_wrapped_if_datetimelike(other)
 
         return method(self, other)
 

pandas/tests/arithmetic/test_numeric.py

@@ -151,6 +151,23 @@ def test_numeric_cmp_string_numexpr_path(self, box_with_array, monkeypatch):
 
 
 class TestNumericArraylikeArithmeticWithDatetimeLike:
+    def test_mul_timedelta_list(self, box_with_array):
+        # GH#62524
+        box = box_with_array
+        left = np.array([3, 4])
+        left = tm.box_expected(left, box)
+
+        right = [Timedelta(days=1), Timedelta(days=2)]
+
+        result = left * right
+
+        expected = TimedeltaIndex([Timedelta(days=3), Timedelta(days=8)])
+        expected = tm.box_expected(expected, box)
+        tm.assert_equal(result, expected)
+
+        result2 = right * left
+        tm.assert_equal(result2, expected)
+
     @pytest.mark.parametrize("box_cls", [np.array, Index, Series])
     @pytest.mark.parametrize(
         "left", lefts, ids=lambda x: type(x).__name__ + str(x.dtype)

pandas/tests/arithmetic/test_timedelta64.py

@@ -274,6 +274,23 @@ class TestTimedelta64ArithmeticUnsorted:
     # Tests moved from type-specific test files but not
     #  yet sorted/parametrized/de-duplicated
 
+    def test_td64_op_with_list(self, box_with_array):
+        # GH#62353
+        box = box_with_array
+
+        left = TimedeltaIndex(["2D", "4D"])
+        left = tm.box_expected(left, box)
+
+        right = [Timestamp("2016-01-01"), Timestamp("2016-02-01")]
+
+        result = left + right
+        expected = DatetimeIndex(["2016-01-03", "2016-02-05"], dtype="M8[ns]")
+        expected = tm.box_expected(expected, box)
+        tm.assert_equal(result, expected)
+
+        result2 = right + left
+        tm.assert_equal(result2, expected)
+
     def test_ufunc_coercions(self):
         # normal ops are also tested in tseries/test_timedeltas.py
         idx = TimedeltaIndex(["2h", "4h", "6h", "8h", "10h"], freq="2h", name="x")

pandas/tests/arrays/string_/test_string.py

@@ -218,7 +218,15 @@ def test_add_2d(dtype, request):
         s + b
 
 
-def test_add_sequence(dtype):
+def test_add_sequence(dtype, request):
+    if dtype.storage == "python" and dtype.na_value is np.nan:
+        mark = pytest.mark.xfail(
+            reason="As of GH#62522, the list gets wrapped withm sanitize_array, "
+            "which casts to a higher-priority StringArray, so we get "
+            "NotImplemented."
+        )
+        request.applymarker(mark)
+
     a = pd.array(["a", "b", None, None], dtype=dtype)
     other = ["x", None, "y", None]
 
@@ -391,10 +399,18 @@ def test_comparison_methods_array_arrow_extension(comparison_op, dtype2):
     tm.assert_extension_array_equal(result, expected)
 
 
-def test_comparison_methods_list(comparison_op, dtype):
+@pytest.mark.parametrize("box", [pd.array, pd.Index, pd.Series])
+def test_comparison_methods_list(comparison_op, dtype, box, request):
+    if box is pd.array and dtype.na_value is np.nan:
+        mark = pytest.mark.xfail(
+            reason="After wrapping list, op returns NotImplemented, see GH#62522"
+        )
+        request.applymarker(mark)
+
     op_name = f"__{comparison_op.__name__}__"
 
-    a = pd.array(["a", None, "c"], dtype=dtype)
+    a = box(pd.array(["a", None, "c"], dtype=dtype))
+    item = "c"
     other = [None, None, "c"]
     result = comparison_op(a, other)
 
@@ -407,15 +423,21 @@ def test_comparison_methods_list(comparison_op, dtype):
             expected = np.array([True, True, False])
         else:
             expected = np.array([False, False, False])
-            expected[-1] = getattr(other[-1], op_name)(a[-1])
-        tm.assert_numpy_array_equal(result, expected)
+            expected[-1] = getattr(other[-1], op_name)(item)
+        if box is not pd.Index:
+            # if GH#62766 is addressed this check can be removed
+            expected = tm.box_expected(expected, box)
+        tm.assert_equal(result, expected)
 
     else:
         expected_dtype = "boolean[pyarrow]" if dtype.storage == "pyarrow" else "boolean"
         expected = np.full(len(a), fill_value=None, dtype="object")
-        expected[-1] = getattr(other[-1], op_name)(a[-1])
+        expected[-1] = getattr(other[-1], op_name)(item)
         expected = pd.array(expected, dtype=expected_dtype)
-        tm.assert_extension_array_equal(result, expected)
+        if box is not pd.Index:
+            # if GH#62766 is addressed this check can be removed
+            expected = tm.box_expected(expected, box)
+        tm.assert_equal(result, expected)
 
 
 def test_constructor_raises(cls):

pandas/tests/indexes/categorical/test_category.py

@@ -326,10 +326,9 @@ def test_disallow_addsub_ops(self, func, op_name):
         cat_or_list = "'(Categorical|list)' and '(Categorical|list)'"
         msg = "|".join(
             [
-                f"cannot perform {op_name} with this index type: CategoricalIndex",
-                "can only concatenate list",
                 rf"unsupported operand type\(s\) for [\+-]: {cat_or_list}",
-                "Object with dtype category cannot perform the numpy op ",
+                "operation 'r?(add|sub)' not supported for dtype 'str' "
+                "with dtype 'category'",
             ]
         )
         with pytest.raises(TypeError, match=msg):

pandas/tests/series/test_arithmetic.py

@@ -358,10 +358,10 @@ def test_mask_div_propagate_na_for_non_na_dtype(self):
 
     @pytest.mark.parametrize("val", [3, 3.5])
     def test_add_list_to_masked_array(self, val):
-        # GH#22962
+        # GH#22962, behavior changed by GH#62552
         ser = Series([1, None, 3], dtype="Int64")
         result = ser + [1, None, val]
-        expected = Series([2, pd.NA, 3 + val], dtype=object)
+        expected = Series([2, pd.NA, 3 + val], dtype="Float64")
         tm.assert_series_equal(result, expected)
 
         result = [1, None, val] + ser

pandas/tests/series/test_constructors.py

@@ -2216,7 +2216,16 @@ def test_np_string_array_object_cast(self, data):
         arr = np.array(data, dtype=StringDType())
         res = Series(arr)
         assert res.dtype == np.object_
-        assert (res == data).all()
+
+        if data[-1] is np.nan:
+            # as of GH#62522 the comparison op for `res==data` casts data
+            #  using sanitize_array, which casts to 'str' dtype, which does not
+            #  consider string 'nan' to be equal to np.nan,
+            #  (which apparently numpy does?  weird.)
+            assert (res.iloc[:-1] == data[:-1]).all()
+            assert res.iloc[-1] == "nan"
+        else:
+            assert (res == data).all()
 
 
 class TestSeriesConstructorInternals: