Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Sign up
Appearance settings

BUG: Fix incorrect FutureWarning for logical ops on pyarrow bool Series (#62260) #62290

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
1d53ce7
replacing null with false for logical operations
Tarun2605 Sep 7, 2025
83a8069
Conditional Null filling by checking if pyarrow is availble or now
Tarun2605 Sep 8, 2025
8c90b35
Filling null values with false for boolean arrays only
Tarun2605 Sep 8, 2025
c2922e7
Filling nulls when it is arrow bit functions
Tarun2605 Sep 8, 2025
3492609
Fixing Bool which were previously not aligning with Kleene's principl...
Tarun2605 Sep 9, 2025
a6f12a0
Changed a few test cases to align with kleene principle
Tarun2605 Sep 10, 2025
adbedf9
Update array_ops.py to pass dosctring check
Tarun2605 Sep 10, 2025
95e06d3
Update array_ops.py
Tarun2605 Sep 10, 2025
553f1fc
Update array_ops.py
Tarun2605 Sep 10, 2025
17eaf2c
Correction for Code Checks / Docstring validation, typing, and other ...
Tarun2605 Sep 10, 2025
8bbb3b0
Fix: safe_is_true now returns True only for exact boolean True, not f...
Tarun2605 Sep 10, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
119 changes: 118 additions & 1 deletion pandas/core/ops/array_ops.py
View file Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -389,6 +389,120 @@ def na_logical_op(x: np.ndarray, y, op):
return result.reshape(x.shape)


def is_nullable_bool(arr) -> bool:
if isinstance(arr, np.ndarray):
if arr.size == 0:
return True

arr = np.asarray(arr, dtype=object).ravel()
# isna works elementwise on object arrays
na_mask = isna(arr)
bool_mask = np.array([x is True or x is False for x in arr])
return bool(np.all(na_mask | bool_mask))


def safe_is_true(arr: np.ndarray) -> np.ndarray:
# Identify missing values (NA, NaN, None, etc.)
mask = isna(arr)

# Prepare boolean output with the same shape as input
out = np.zeros(arr.shape, dtype=bool)

# Flatten for uniform indexing regardless of ndim
flat_arr = arr.ravel()
flat_mask = mask.ravel()
flat_out = out.ravel()

# Only compare non-missing values against True
valid = ~flat_mask

flat_out[valid] = [x is True for x in flat_arr[valid]]

return out


def alignOutputWithKleene(left, right, op):
"""
Apply Kleene's 3-valued logic (with NA) to elementwise boolean operations.

Parameters
----------
left, right : array-like
Input arrays containing True, False, or NA (np.nan/pd.NA/None).
op : function
Operator function from the operator module, e.g. operator.and_,
operator.or_, operator.xor.

Returns
-------
result : np.ndarray
Array with elements True, False, or np.nan (for NA).
Uses bool dtype if no NA, otherwise object dtype.
"""
left = np.asarray(left, dtype=object)
right = np.asarray(right, dtype=object)

# Masks for NA values
left_mask = isna(left)
right_mask = isna(right)

# Boolean arrays ignoring NA
lvalues = safe_is_true(left)
rvalues = safe_is_true(right)
# lvalues = (left == True) & ~left_mask
# rvalues = (right == True) & ~right_mask

# Initialize result
res_values = np.empty_like(left, dtype=bool)
mask = np.zeros_like(left, dtype=bool)

# --- AND logic ---
# Special case: all-NA inputs (e.g. dfa & dfa)
if op.__name__ in {"and_", "rand_"} and left_mask.all() and right_mask.all():
result = np.zeros_like(res_values, dtype=bool) # all False, bool dtype
return result

if op.__name__ in {"and_", "rand_"}:
res_values[:] = lvalues & rvalues
mask[:] = (
(left_mask & rvalues) | (right_mask & lvalues) | (left_mask & right_mask)
)

# --- OR logic ---
elif op.__name__ in {"or_", "ror_"}:
res_values[:] = lvalues | rvalues
# Unknown only if both sides are NA
mask[:] = left_mask & right_mask

# Handle cases where NA OR False → False, NA OR True → True
# Pandas convention: np.nan | False -> False, np.nan | True -> True
res_values[left_mask & ~rvalues] = False
res_values[right_mask & ~lvalues] = False
res_values[left_mask & rvalues] = True
res_values[right_mask & lvalues] = True

# --- XOR logic ---
elif op.__name__ in {"xor", "rxor"}:
res_values[:] = lvalues ^ rvalues
mask[:] = left_mask | right_mask

else:
raise ValueError(f"Unsupported operator: {op.__name__}")

# Apply mask → insert np.nan only if needed
if mask.any():
result = res_values.astype(object)
result[mask] = np.nan
else:
result = res_values.astype(bool)

# Handle empty arrays explicitly to satisfy pandas dtype expectations
if result.size == 0:
result = result.astype(bool)

return result


def logical_op(left: ArrayLike, right: Any, op) -> ArrayLike:
"""
Evaluate a logical operation `|`, `&`, or `^`.
Expand Down Expand Up @@ -449,12 +563,15 @@ def fill_bool(x, left=None):
is_other_int_dtype = lib.is_integer(rvalues)

res_values = na_logical_op(lvalues, rvalues, op)
bothAreBoolArrays = is_nullable_bool(left) and is_nullable_bool(right)
# print("Yes both are bools", bothAreBoolArrays)
if bothAreBoolArrays:
return alignOutputWithKleene(left, right, op)

# For int vs int `^`, `|`, `&` are bitwise operators and return
# integer dtypes. Otherwise these are boolean ops
if not (left.dtype.kind in "iu" and is_other_int_dtype):
res_values = fill_bool(res_values)

return res_values


Expand Down
31 changes: 24 additions & 7 deletions pandas/tests/frame/test_logical_ops.py
View file Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,19 +24,31 @@ class TestDataFrameLogicalOperators:
[True, False, np.nan],
[True, False, True],
operator.and_,
[True, False, False],
[
True,
False,
np.nan,
], # changed last element, Kleene AND with Unknown gives Unknown
),
(
[True, False, True],
[True, False, np.nan],
operator.and_,
[True, False, False],
[
True,
False,
np.nan,
], # changed last element, Kleene AND with Unknown gives Unknown
),
(
[True, False, np.nan],
[True, False, True],
operator.or_,
[True, False, False],
[
True,
False,
True,
], # change last element, Kleene Or of True and unknown gives true
),
(
[True, False, True],
Expand Down Expand Up @@ -157,16 +169,21 @@ def _check_unary_op(op):
def test_logical_with_nas(self):
d = DataFrame({"a": [np.nan, False], "b": [True, True]})

# GH4947
# bool comparisons should return bool
# In Kleene logic:
# NaN OR True → True
# False OR True → True
result = d["a"] | d["b"]
expected = Series([False, True])
expected = Series([True, True])
tm.assert_series_equal(result, expected)

# GH4604, automatic casting here
# If we explicitly fill NaN with False first:
# row0: False OR True → True
# row1: False OR True → True
result = d["a"].fillna(False) | d["b"]
expected = Series([True, True])
tm.assert_series_equal(result, expected)

# Redundant check (same as above)
result = d["a"].fillna(False) | d["b"]
expected = Series([True, True])
tm.assert_series_equal(result, expected)
Expand Down
42 changes: 23 additions & 19 deletions pandas/tests/series/test_logical_ops.py
View file Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,11 +37,12 @@ def test_logical_operators_bool_dtype_with_empty(self):
index = list("bca")

s_tft = Series([True, False, True], index=index)
s_fff = Series([False, False, False], index=index)
# s_fff = Series([False, False, False], index=index)
s_empty = Series([], dtype=object)

res = s_tft & s_empty
expected = s_fff.sort_index()
# changed the test case output to align with kleene principle
expected = Series([np.nan, False, np.nan], index=index).sort_index()
tm.assert_series_equal(res, expected)

res = s_tft | s_empty
Expand Down Expand Up @@ -180,8 +181,8 @@ def test_logical_ops_bool_dtype_with_ndarray(self):
r"Logical ops \(and, or, xor\) between Pandas objects and "
"dtype-less sequences"
)

expected = Series([True, False, False, False, False])
# changed the test case output to align with kleene principle
expected = Series([True, False, np.nan, False, np.nan])
with pytest.raises(TypeError, match=msg):
left & right
result = left & np.array(right)
Expand All @@ -200,8 +201,8 @@ def test_logical_ops_bool_dtype_with_ndarray(self):
tm.assert_series_equal(result, expected)
result = left | Series(right)
tm.assert_series_equal(result, expected)

expected = Series([False, True, True, True, True])
# changed the test case output to align with kleene principle
expected = Series([False, True, np.nan, True, np.nan])
with pytest.raises(TypeError, match=msg):
left ^ right
result = left ^ np.array(right)
Expand Down Expand Up @@ -368,12 +369,12 @@ def test_logical_ops_label_based(self, using_infer_string):
# rhs is bigger
a = Series([True, False, True], list("bca"))
b = Series([False, True, False, True], list("abcd"))

expected = Series([False, True, False, False], list("abcd"))
# changed the test case output to align with kleene principle
expected = Series([False, True, False, np.nan], list("abcd"))
result = a & b
tm.assert_series_equal(result, expected)

expected = Series([True, True, False, False], list("abcd"))
# changed the test case output to align with kleene principle
expected = Series([True, True, False, True], list("abcd"))
result = a | b
tm.assert_series_equal(result, expected)

Expand All @@ -383,7 +384,8 @@ def test_logical_ops_label_based(self, using_infer_string):
empty = Series([], dtype=object)

result = a & empty
expected = Series([False, False, False], list("abc"))
# changed the test case output to align with kleene principle
expected = Series([np.nan, np.nan, False], list("abc"))
tm.assert_series_equal(result, expected)

result = a | empty
Expand All @@ -407,7 +409,9 @@ def test_logical_ops_label_based(self, using_infer_string):
Series(np.nan, b.index),
Series(np.nan, a.index),
]:
result = a[a | e]
result = a[(a | e).astype("boolean")]
# cast to boolean because object dtype with nan
# cannot be compared to True
tm.assert_series_equal(result, a[a])

for e in [Series(["z"])]:
Expand Down Expand Up @@ -459,16 +463,16 @@ def test_logical_ops_df_compat(self):
# GH#1134
s1 = Series([True, False, True], index=list("ABC"), name="x")
s2 = Series([True, True, False], index=list("ABD"), name="x")

exp = Series([True, False, False, False], index=list("ABCD"), name="x")
# changed the test case output to align with kleene principle
exp = Series([True, False, np.nan, False], index=list("ABCD"), name="x")
tm.assert_series_equal(s1 & s2, exp)
tm.assert_series_equal(s2 & s1, exp)

# True | np.nan => True
exp_or1 = Series([True, True, True, False], index=list("ABCD"), name="x")
tm.assert_series_equal(s1 | s2, exp_or1)
# np.nan | True => np.nan, filled with False
exp_or = Series([True, True, False, False], index=list("ABCD"), name="x")
# np.nan | True => True (should be)
exp_or = Series([True, True, True, False], index=list("ABCD"), name="x")
tm.assert_series_equal(s2 | s1, exp_or)

# DataFrame doesn't fill nan with False
Expand All @@ -482,13 +486,13 @@ def test_logical_ops_df_compat(self):
# different length
s3 = Series([True, False, True], index=list("ABC"), name="x")
s4 = Series([True, True, True, True], index=list("ABCD"), name="x")

exp = Series([True, False, True, False], index=list("ABCD"), name="x")
# changed the test case output to align with kleene principle
exp = Series([True, False, True, np.nan], index=list("ABCD"), name="x")
tm.assert_series_equal(s3 & s4, exp)
tm.assert_series_equal(s4 & s3, exp)

# np.nan | True => np.nan, filled with False
exp_or1 = Series([True, True, True, False], index=list("ABCD"), name="x")
exp_or1 = Series([True, True, True, True], index=list("ABCD"), name="x")
tm.assert_series_equal(s3 | s4, exp_or1)
# True | np.nan => True
exp_or = Series([True, True, True, True], index=list("ABCD"), name="x")
Expand Down
Loading

AltStyle によって変換されたページ (->オリジナル) /