Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
24 commits
Select commit Hold shift + click to select a range
f8d3b11
fix(bigtable): resolve O(n^2) bottleneck in ReadRowsQuery.add_range
parthea Oct 15, 2026
679ebed
address offline feedback
parthea Oct 15, 2026
285b2c1
update comment
parthea Oct 15, 2026
01e759a
fix build; add test
parthea Oct 15, 2026
d08cc34
address review feedback
parthea Oct 15, 2026
b864707
move import
parthea Oct 15, 2026
3f54cac
Merge remote-tracking branch 'origin/main' into fix-bigtable-20261015
parthea Oct 17, 2026
ef09fca
fix(bigtable): align RowRange.__hash__ with __eq__ across base and le…
parthea Oct 17, 2026
183df93
address review feedback
parthea Oct 17, 2026
39ddcf0
address offline feedback
parthea Oct 17, 2026
f9b35b1
fix(bigtable): store immutable tuple signatures in _range_signatures …
parthea Oct 19, 2026
213fa5b
perf(bigtable): resolve O(n^2) in ReadRowsQuery.add_key and __eq__, e…
parthea Oct 19, 2026
bb51980
fix build
parthea Oct 19, 2026
d2b8192
mypy
parthea Oct 19, 2026
bff7b0b
address review feedback
parthea Oct 19, 2026
09110b3
lint
parthea Oct 19, 2026
26ac51f
Merge remote-tracking branch 'origin/main' into fix-bigtable-20261015
parthea Oct 19, 2026
73f1e1c
fix(bigtable): distinguish unset and empty-byte bounds in RowRange._s…
parthea Oct 19, 2026
6d94d4f
test(bigtable): tighten mock and mutation assertions in ReadRowsQuery…
parthea Oct 19, 2026
8e32f9c
perf(bigtable): enforce __slots__ on legacy row_set.RowRange
parthea Oct 19, 2026
3491e36
perf(bigtable): avoid RowRange wrapper allocation in add_range(RowRan…
parthea Oct 19, 2026
d88fecb
run nox -s generate_sync
parthea Oct 19, 2026
0f285aa
address review feedback
parthea Oct 19, 2026
64125aa
lint
parthea Oct 19, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,7 @@ class _ReadRowsOperationAsync:

def __init__(
self,
query: ReadRowsQuery,
query: ReadRowsQuery | dict[str, object],
target: TargetType,
operation_timeout: float,
attempt_timeout: float,
Expand All @@ -99,6 +99,10 @@ def __init__(
attempt_timeout, operation_timeout
)
self.operation_timeout = operation_timeout
# ReadRowsQuery defines __slots__, which gives it a C-level memory layout
# disjoint from dict. Consequently, `query` must be explicitly typed as
# `ReadRowsQuery | dict[str, object]` in the signature above; otherwise,
# mypy flags this runtime `isinstance(query, dict)` branch as unreachable.

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: Is this comment really useful? It seems like it would feel out of place after the PR is merged

if isinstance(query, dict):
self.request = ReadRowsRequestPB(
**query,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,7 @@ class _ReadRowsOperation:

def __init__(
self,
query: ReadRowsQuery,
query: ReadRowsQuery | dict[str, object],
target: TargetType,
operation_timeout: float,
attempt_timeout: float,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,59 @@ def __init__(
init_dict["end_key_open"] = end_key
self._pb = RowRangePB(**init_dict)

@staticmethod
def _signature_from_pb(
pb: RowRangePB,
) -> tuple[bytes | None, bool, bytes | None, bool]:

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: if we do keep this, we should probably add a type alias for the signature

raw_pb = pb._pb

# Fast path: non-empty keys bypass HasField and proto-plus overhead
start_closed = raw_pb.start_key_closed
if start_closed:
start_key: bytes | None = start_closed
start_is_inclusive = True
else:
start_open = raw_pb.start_key_open
if start_open:
start_key = start_open
start_is_inclusive = False
elif raw_pb.HasField("start_key_closed"):
start_key = b""
start_is_inclusive = True
elif raw_pb.HasField("start_key_open"):
start_key = b""
start_is_inclusive = False
else:
start_key = None
start_is_inclusive = True

end_open = raw_pb.end_key_open
if end_open:
end_key: bytes | None = end_open
end_is_inclusive = False
else:
end_closed = raw_pb.end_key_closed
if end_closed:
end_key = end_closed
end_is_inclusive = True
elif raw_pb.HasField("end_key_open"):
end_key = b""
end_is_inclusive = False
elif raw_pb.HasField("end_key_closed"):
end_key = b""
end_is_inclusive = True
else:
end_key = None
end_is_inclusive = False
Comment on lines +96 to +133

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

medium

The current nested conditional logic with HasField checks is complex and prone to errors when handling edge cases like empty bytes (b""). Since start_key and end_key are defined as oneof fields in the protobuf definition, we can use the standard WhichOneof method on the underlying protobuf message. This simplifies the logic significantly, improves readability, and is more idiomatic.

        start_oneof = raw_pb.WhichOneof("start_key")
        if start_oneof == "start_key_closed":
            start_key = raw_pb.start_key_closed
            start_is_inclusive = True
        elif start_oneof == "start_key_open":
            start_key = raw_pb.start_key_open
            start_is_inclusive = False
        else:
            start_key = None
            start_is_inclusive = True

        end_oneof = raw_pb.WhichOneof("end_key")
        if end_oneof == "end_key_open":
            end_key = raw_pb.end_key_open
            end_is_inclusive = False
        elif end_oneof == "end_key_closed":
            end_key = raw_pb.end_key_closed
            end_is_inclusive = True
        else:
            end_key = None
            end_is_inclusive = False

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Keeping the current structure here intentionally as a hot-path optimization.

See analysis below:

In the common case (non-empty start_key_closed and end_key_open), checking truthiness directly on raw_pb.start_key_closed and raw_pb.end_key_open avoids extra WhichOneof method call and string comparison overhead on every add_range call, only falling back to HasField when a bound is empty (b"") or unset (None).

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

IIUC, we shouldn't need the HasField fallback case at all. Bigtable treats falsy values as unbounded keys, and the handwritten classes collapse them into None


return (start_key, start_is_inclusive, end_key, end_is_inclusive)

def _signature(self) -> tuple[bytes | None, bool, bytes | None, bool]:
return self._signature_from_pb(self._pb)

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

_signature_from_pb seems very complex. Isn't this doing the same job as the legacy client's _key()? Can't we just move _key() and __hash__() into the base class here, and avoid the complex parsing logic?

(Also, are you sure we need both _signature and _signature_from_pb? I only see _signature_from_pb being used to handle PB inputs, but I don't think we'd actually want to expose that publicly)


def __hash__(self) -> int:

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

__eq__ compares _pb, but __hash__ hashes self._key(). The legacy subclass google.cloud.bigtable.row_set.RowRange overrides _key() with a different field order, so legacy and base ranges compare equal but hash differently. That breaks set membership for mixed types, and dedup regresses vs main:

from google.cloud.bigtable.row_set import RowRange as LegacyRowRange

b, l = RowRange(b"a", b"b"), LegacyRowRange(b"a", b"b")
b == l               # True
hash(b) == hash(l)   # False

q = ReadRowsQuery(row_ranges=[b])
q.add_range(l)
len(q.row_ranges)    # 2 on this branch, 1 on main

Suggest hashing the same data __eq__ compares:

def __hash__(self) -> int:
    pb = self._pb
    return hash(
        (pb.start_key_closed, pb.start_key_open, pb.end_key_closed, pb.end_key_open)
    )

and dropping __hash__ from row_set.RowRange so it inherits this one (_key() then isn't needed here). I tried this locally: mixed legacy/base dedup works, the bigtable unit tests pass, and 10k add_range calls still take ~0.23s. Could you add a test along these lines?

def test_row_range_hash_matches_legacy(self):
    from google.cloud.bigtable.data.read_rows_query import RowRange
    from google.cloud.bigtable.row_set import RowRange as LegacyRowRange

    b, l = RowRange(b"a", b"b"), LegacyRowRange(b"a", b"b")
    assert b == l
    assert hash(b) == hash(l)
    query = self._make_one(row_ranges=[b])
    query.add_range(l)
    assert len(query.row_ranges) == 1

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Done in ef09fca.

RowRange.__hash__ now hashes the 4 proto fields (via _signature()), row_set.RowRange.__hash__ is removed so it inherits the base implementation, and I added test_row_range_hash_matches_legacy.

return hash(self._signature())

@property
def start_key(self) -> bytes | None:
"""
Expand Down Expand Up @@ -147,7 +200,8 @@ def _from_pb(cls, data: RowRangePB) -> RowRange:
Returns:
RowRange: The converted RowRange
"""
instance = cls()
# Skip __init__ to avoid allocating an unused RowRangePB
instance = cls.__new__(cls)
instance._pb = data
return instance

Expand All @@ -164,7 +218,8 @@ def _from_dict(cls, data: dict[str, bytes | str]) -> RowRange:
formatted_data = {
k: v.encode() if isinstance(v, str) else v for k, v in data.items()
}
instance = cls()
# Skip __init__ to avoid allocating an unused RowRangePB
instance = cls.__new__(cls)
instance._pb = RowRangePB(**formatted_data)
return instance

Expand Down Expand Up @@ -229,7 +284,13 @@ class ReadRowsQuery:
row_filter: a RowFilter to apply to the query
"""

slots = ("_limit", "_filter", "_row_set")
__slots__ = (
"_limit",
"_filter",
"_row_set",
"_key_set",
"_range_signatures",
)

def __init__(
self,
Expand All @@ -246,10 +307,15 @@ def __init__(
row_ranges = [row_ranges]
if not isinstance(row_keys, list):
row_keys = [row_keys]
row_keys = [key.encode() if isinstance(key, str) else key for key in row_keys]
self._row_set = RowSetPB(
row_keys=row_keys, row_ranges=[r._pb for r in row_ranges]
self._row_set = RowSetPB()

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It seems like we're duplicating some state now, if we are tracking both row_set, and keys/ranges separately. Lots of room for those to get out of sync

Could we remove row_set, turn range_signatures into a set of full Range instances, and just re-construct the row_set inside to_pb? That seems like it could simplify the data model. Or am I misunderstanding?

self._key_set: set[bytes] = set()
self._range_signatures: set[tuple[bytes | None, bool, bytes | None, bool]] = (
set()
)
for k in row_keys:
self.add_key(k)
for r in row_ranges:
self.add_range(r)
self.limit = limit or None
self.filter = row_filter

Expand Down Expand Up @@ -336,21 +402,29 @@ def add_key(self, row_key: str | bytes):
row_key = row_key.encode()
elif not isinstance(row_key, bytes):
raise ValueError("row_key must be string or bytes")
if row_key not in self._row_set.row_keys:
if row_key not in self._key_set:
self._key_set.add(row_key)
self._row_set.row_keys.append(row_key)

def add_range(
self,
row_range: RowRange,
row_range: RowRange | RowRangePB,

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is this necessary? Bigtable wanted to hide protobufs from the public interface

):
"""
Add a range of row keys to this query.
Args:
row_range: a range of row keys to add to this query
"""
if row_range not in self.row_ranges:
self._row_set.row_ranges.append(row_range._pb)
if isinstance(row_range, RowRangePB):
pb = row_range
sig = RowRange._signature_from_pb(pb)
else:
pb = row_range._pb
sig = row_range._signature()
if sig not in self._range_signatures:
self._range_signatures.add(sig)
self._row_set.row_ranges.append(pb)

def shard(self, shard_keys: RowKeySamples) -> ShardedQuery:
"""
Expand Down Expand Up @@ -495,6 +569,15 @@ def _to_pb(self, table) -> ReadRowsRequestPB:
**table._request_path,
)

def __copy__(self) -> ReadRowsQuery:
new_query = ReadRowsQuery(limit=self.limit, row_filter=self.filter)
type(self._row_set).pb(new_query._row_set).CopyFrom(
type(self._row_set).pb(self._row_set)
)
new_query._key_set = self._key_set.copy()
new_query._range_signatures = self._range_signatures.copy()
return new_query
Comment thread
parthea marked this conversation as resolved.

def __eq__(self, other):
"""
RowRanges are equal if they have the same row keys, row ranges,
Expand All @@ -509,24 +592,24 @@ def __eq__(self, other):
if not isinstance(other, ReadRowsQuery):
return False
# empty queries are equal
if len(self.row_keys) == 0 and len(other.row_keys) == 0:
this_range_empty = len(self.row_ranges) == 0 or all(
[bool(r) is False for r in self.row_ranges]
if len(self._key_set) == 0 and len(other._key_set) == 0:
empty_sig = (None, True, None, False)
this_range_empty = len(self._range_signatures) == 0 or (
self._range_signatures == {empty_sig}
)
other_range_empty = len(other.row_ranges) == 0 or all(
[bool(r) is False for r in other.row_ranges]
other_range_empty = len(other._range_signatures) == 0 or (
other._range_signatures == {empty_sig}
)
if this_range_empty and other_range_empty:
return self.filter == other.filter and self.limit == other.limit
# otherwise, sets should have same sizes
if len(self.row_keys) != len(other.row_keys):
if len(self._key_set) != len(other._key_set):
return False
if len(self.row_ranges) != len(other.row_ranges):
if len(self._range_signatures) != len(other._range_signatures):
return False
ranges_match = all([row in other.row_ranges for row in self.row_ranges])
return (
self.row_keys == other.row_keys
and ranges_match
self._key_set == other._key_set
and self._range_signatures == other._range_signatures
and self.filter == other.filter
and self.limit == other.limit
)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,8 @@ class and define a map _attribute_map from legacy client attributes to data clie
Attributes are remapped and redefined in __init__ as well as getattr/setattr.
"""

__slots__ = ()

def __init__(self, *args, **kwargs):
new_kwargs = {self._attribute_map.get(k, k): v for (k, v) in kwargs.items()}
super(_MappableAttributesMixin, self).__init__(*args, **new_kwargs)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -152,6 +152,8 @@ class RowRange(_MappableAttributesMixin, BaseRowRange):
considered inclusive. The default is False (exclusive).
"""

__slots__ = ()

_attribute_map = {
"start_inclusive": "start_is_inclusive",
"end_inclusive": "end_is_inclusive",
Expand All @@ -165,9 +167,6 @@ def _key(self):
self.end_is_inclusive,
)

def __hash__(self):
return hash(self._key())

def get_range_kwargs(self):
"""Convert row range object to dict which can be passed to
google.bigtable.v2.RowRange add method.
Expand Down
Loading
Loading