Backport PR #40010: REGR: compressed to_json with URL-like paths and binary objects (#40040)

Author: twoertwein

doc/source/whatsnew/v1.2.3.rst

@@ -18,6 +18,7 @@ Fixed regressions
 - Fixed regression in :meth:`~DataFrame.to_excel` raising ``KeyError`` when giving duplicate columns with ``columns`` attribute (:issue:`39695`)
 - Fixed regression in nullable integer unary ops propagating mask on assignment (:issue:`39943`)
 - Fixed regression in :meth:`DataFrame.__setitem__` not aligning :class:`DataFrame` on right-hand side for boolean indexer (:issue:`39931`)
+- Fixed regression in :meth:`~DataFrame.to_json` failing to use ``compression`` with URL-like paths that are internally opened in binary mode or with user-provided file objects that are opened in binary mode (:issue:`39985`)
 
 .. ---------------------------------------------------------------------------
 

pandas/io/json/_json.py

@@ -100,7 +100,7 @@ def to_json(
     if path_or_buf is not None:
         # apply compression and byte/text conversion
         with get_handle(
-            path_or_buf, "wt", compression=compression, storage_options=storage_options
+            path_or_buf, "w", compression=compression, storage_options=storage_options
         ) as handles:
             handles.handle.write(s)
     else:

pandas/tests/io/json/test_compression.py

@@ -1,3 +1,5 @@
+from io import BytesIO
+
 import pytest
 
 import pandas.util._test_decorators as td
@@ -115,3 +117,13 @@ def test_to_json_compression(compression_only, read_infer, to_infer):
         df.to_json(path, compression=to_compression)
         result = pd.read_json(path, compression=read_compression)
         tm.assert_frame_equal(result, df)
+
+
+def test_to_json_compression_mode(compression):
+    # GH 39985 (read_json does not support user-provided binary files)
+    expected = pd.DataFrame({"A": [1]})
+
+    with BytesIO() as buffer:
+        expected.to_json(buffer, compression=compression)
+        # df = pd.read_json(buffer, compression=compression)
+        # tm.assert_frame_equal(expected, df)

pandas/tests/io/test_fsspec.py

@@ -249,11 +249,19 @@ def test_pickle_options(fsspectest):
     tm.assert_frame_equal(df, out)
 
 
-def test_json_options(fsspectest):
+def test_json_options(fsspectest, compression):
     df = DataFrame({"a": [0]})
-    df.to_json("testmem://afile", storage_options={"test": "json_write"})
+    df.to_json(
+        "testmem://afile",
+        compression=compression,
+        storage_options={"test": "json_write"},
+    )
     assert fsspectest.test[0] == "json_write"
-    out = read_json("testmem://afile", storage_options={"test": "json_read"})
+    out = read_json(
+        "testmem://afile",
+        compression=compression,
+        storage_options={"test": "json_read"},
+    )
     assert fsspectest.test[0] == "json_read"
     tm.assert_frame_equal(df, out)