Repository navigation
fix(bigtable): resolve O(n^2) bottleneck in ReadRowsQuery.add_range #18706
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
f8d3b11
679ebed
285b2c1
01e759a
d08cc34
b864707
3f54cac
ef09fca
183df93
39ddcf0
f9b35b1
213fa5b
bb51980
d2b8192
bff7b0b
09110b3
26ac51f
73f1e1c
6d94d4f
8e32f9c
3491e36
d88fecb
0f285aa
64125aa
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -87,6 +87,59 @@ def __init__( | |
| init_dict["end_key_open"] = end_key | ||
| self._pb = RowRangePB(**init_dict) | ||
|
|
||
| @staticmethod | ||
| def _signature_from_pb( | ||
| pb: RowRangePB, | ||
| ) -> tuple[bytes | None, bool, bytes | None, bool]: | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. nit: if we do keep this, we should probably add a type alias for the signature |
||
| raw_pb = pb._pb | ||
|
|
||
| # Fast path: non-empty keys bypass HasField and proto-plus overhead | ||
| start_closed = raw_pb.start_key_closed | ||
| if start_closed: | ||
| start_key: bytes | None = start_closed | ||
| start_is_inclusive = True | ||
| else: | ||
| start_open = raw_pb.start_key_open | ||
| if start_open: | ||
| start_key = start_open | ||
| start_is_inclusive = False | ||
| elif raw_pb.HasField("start_key_closed"): | ||
| start_key = b"" | ||
| start_is_inclusive = True | ||
| elif raw_pb.HasField("start_key_open"): | ||
| start_key = b"" | ||
| start_is_inclusive = False | ||
| else: | ||
| start_key = None | ||
| start_is_inclusive = True | ||
|
|
||
| end_open = raw_pb.end_key_open | ||
| if end_open: | ||
| end_key: bytes | None = end_open | ||
| end_is_inclusive = False | ||
| else: | ||
| end_closed = raw_pb.end_key_closed | ||
| if end_closed: | ||
| end_key = end_closed | ||
| end_is_inclusive = True | ||
| elif raw_pb.HasField("end_key_open"): | ||
| end_key = b"" | ||
| end_is_inclusive = False | ||
| elif raw_pb.HasField("end_key_closed"): | ||
| end_key = b"" | ||
| end_is_inclusive = True | ||
| else: | ||
| end_key = None | ||
| end_is_inclusive = False | ||
|
Comment on lines
+96
to
+133
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. The current nested conditional logic with start_oneof = raw_pb.WhichOneof("start_key")
if start_oneof == "start_key_closed":
start_key = raw_pb.start_key_closed
start_is_inclusive = True
elif start_oneof == "start_key_open":
start_key = raw_pb.start_key_open
start_is_inclusive = False
else:
start_key = None
start_is_inclusive = True
end_oneof = raw_pb.WhichOneof("end_key")
if end_oneof == "end_key_open":
end_key = raw_pb.end_key_open
end_is_inclusive = False
elif end_oneof == "end_key_closed":
end_key = raw_pb.end_key_closed
end_is_inclusive = True
else:
end_key = None
end_is_inclusive = False
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Keeping the current structure here intentionally as a hot-path optimization. See analysis below:
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. IIUC, we shouldn't need the HasField fallback case at all. Bigtable treats falsy values as unbounded keys, and the handwritten classes collapse them into None |
||
|
|
||
| return (start_key, start_is_inclusive, end_key, end_is_inclusive) | ||
|
|
||
| def _signature(self) -> tuple[bytes | None, bool, bytes | None, bool]: | ||
| return self._signature_from_pb(self._pb) | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
(Also, are you sure we need both |
||
|
|
||
| def __hash__(self) -> int: | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
from google.cloud.bigtable.row_set import RowRange as LegacyRowRange
b, l = RowRange(b"a", b"b"), LegacyRowRange(b"a", b"b")
b == l # True
hash(b) == hash(l) # False
q = ReadRowsQuery(row_ranges=[b])
q.add_range(l)
len(q.row_ranges) # 2 on this branch, 1 on mainSuggest hashing the same data def __hash__(self) -> int:
pb = self._pb
return hash(
(pb.start_key_closed, pb.start_key_open, pb.end_key_closed, pb.end_key_open)
)and dropping def test_row_range_hash_matches_legacy(self):
from google.cloud.bigtable.data.read_rows_query import RowRange
from google.cloud.bigtable.row_set import RowRange as LegacyRowRange
b, l = RowRange(b"a", b"b"), LegacyRowRange(b"a", b"b")
assert b == l
assert hash(b) == hash(l)
query = self._make_one(row_ranges=[b])
query.add_range(l)
assert len(query.row_ranges) == 1
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Done in ef09fca.
|
||
| return hash(self._signature()) | ||
|
|
||
| @property | ||
| def start_key(self) -> bytes | None: | ||
| """ | ||
|
|
@@ -147,7 +200,8 @@ def _from_pb(cls, data: RowRangePB) -> RowRange: | |
| Returns: | ||
| RowRange: The converted RowRange | ||
| """ | ||
| instance = cls() | ||
| # Skip __init__ to avoid allocating an unused RowRangePB | ||
| instance = cls.__new__(cls) | ||
| instance._pb = data | ||
| return instance | ||
|
|
||
|
|
@@ -164,7 +218,8 @@ def _from_dict(cls, data: dict[str, bytes | str]) -> RowRange: | |
| formatted_data = { | ||
| k: v.encode() if isinstance(v, str) else v for k, v in data.items() | ||
| } | ||
| instance = cls() | ||
| # Skip __init__ to avoid allocating an unused RowRangePB | ||
| instance = cls.__new__(cls) | ||
| instance._pb = RowRangePB(**formatted_data) | ||
| return instance | ||
|
|
||
|
|
@@ -229,7 +284,13 @@ class ReadRowsQuery: | |
| row_filter: a RowFilter to apply to the query | ||
| """ | ||
|
|
||
| slots = ("_limit", "_filter", "_row_set") | ||
| __slots__ = ( | ||
| "_limit", | ||
| "_filter", | ||
| "_row_set", | ||
| "_key_set", | ||
| "_range_signatures", | ||
| ) | ||
|
|
||
| def __init__( | ||
| self, | ||
|
|
@@ -246,10 +307,15 @@ def __init__( | |
| row_ranges = [row_ranges] | ||
| if not isinstance(row_keys, list): | ||
| row_keys = [row_keys] | ||
| row_keys = [key.encode() if isinstance(key, str) else key for key in row_keys] | ||
| self._row_set = RowSetPB( | ||
| row_keys=row_keys, row_ranges=[r._pb for r in row_ranges] | ||
| self._row_set = RowSetPB() | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. It seems like we're duplicating some state now, if we are tracking both row_set, and keys/ranges separately. Lots of room for those to get out of sync Could we remove |
||
| self._key_set: set[bytes] = set() | ||
| self._range_signatures: set[tuple[bytes | None, bool, bytes | None, bool]] = ( | ||
| set() | ||
| ) | ||
| for k in row_keys: | ||
| self.add_key(k) | ||
| for r in row_ranges: | ||
| self.add_range(r) | ||
| self.limit = limit or None | ||
| self.filter = row_filter | ||
|
|
||
|
|
@@ -336,21 +402,29 @@ def add_key(self, row_key: str | bytes): | |
| row_key = row_key.encode() | ||
| elif not isinstance(row_key, bytes): | ||
| raise ValueError("row_key must be string or bytes") | ||
| if row_key not in self._row_set.row_keys: | ||
| if row_key not in self._key_set: | ||
| self._key_set.add(row_key) | ||
| self._row_set.row_keys.append(row_key) | ||
|
|
||
| def add_range( | ||
| self, | ||
| row_range: RowRange, | ||
| row_range: RowRange | RowRangePB, | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Is this necessary? Bigtable wanted to hide protobufs from the public interface |
||
| ): | ||
| """ | ||
| Add a range of row keys to this query. | ||
| Args: | ||
| row_range: a range of row keys to add to this query | ||
| """ | ||
| if row_range not in self.row_ranges: | ||
| self._row_set.row_ranges.append(row_range._pb) | ||
| if isinstance(row_range, RowRangePB): | ||
| pb = row_range | ||
| sig = RowRange._signature_from_pb(pb) | ||
| else: | ||
| pb = row_range._pb | ||
| sig = row_range._signature() | ||
| if sig not in self._range_signatures: | ||
| self._range_signatures.add(sig) | ||
| self._row_set.row_ranges.append(pb) | ||
|
|
||
| def shard(self, shard_keys: RowKeySamples) -> ShardedQuery: | ||
| """ | ||
|
|
@@ -495,6 +569,15 @@ def _to_pb(self, table) -> ReadRowsRequestPB: | |
| **table._request_path, | ||
| ) | ||
|
|
||
| def __copy__(self) -> ReadRowsQuery: | ||
| new_query = ReadRowsQuery(limit=self.limit, row_filter=self.filter) | ||
| type(self._row_set).pb(new_query._row_set).CopyFrom( | ||
| type(self._row_set).pb(self._row_set) | ||
| ) | ||
| new_query._key_set = self._key_set.copy() | ||
| new_query._range_signatures = self._range_signatures.copy() | ||
| return new_query | ||
|
parthea marked this conversation as resolved.
|
||
|
|
||
| def __eq__(self, other): | ||
| """ | ||
| RowRanges are equal if they have the same row keys, row ranges, | ||
|
|
@@ -509,24 +592,24 @@ def __eq__(self, other): | |
| if not isinstance(other, ReadRowsQuery): | ||
| return False | ||
| # empty queries are equal | ||
| if len(self.row_keys) == 0 and len(other.row_keys) == 0: | ||
| this_range_empty = len(self.row_ranges) == 0 or all( | ||
| [bool(r) is False for r in self.row_ranges] | ||
| if len(self._key_set) == 0 and len(other._key_set) == 0: | ||
| empty_sig = (None, True, None, False) | ||
| this_range_empty = len(self._range_signatures) == 0 or ( | ||
| self._range_signatures == {empty_sig} | ||
| ) | ||
| other_range_empty = len(other.row_ranges) == 0 or all( | ||
| [bool(r) is False for r in other.row_ranges] | ||
| other_range_empty = len(other._range_signatures) == 0 or ( | ||
| other._range_signatures == {empty_sig} | ||
| ) | ||
| if this_range_empty and other_range_empty: | ||
| return self.filter == other.filter and self.limit == other.limit | ||
| # otherwise, sets should have same sizes | ||
| if len(self.row_keys) != len(other.row_keys): | ||
| if len(self._key_set) != len(other._key_set): | ||
| return False | ||
| if len(self.row_ranges) != len(other.row_ranges): | ||
| if len(self._range_signatures) != len(other._range_signatures): | ||
| return False | ||
| ranges_match = all([row in other.row_ranges for row in self.row_ranges]) | ||
| return ( | ||
| self.row_keys == other.row_keys | ||
| and ranges_match | ||
| self._key_set == other._key_set | ||
| and self._range_signatures == other._range_signatures | ||
| and self.filter == other.filter | ||
| and self.limit == other.limit | ||
| ) | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
nit: Is this comment really useful? It seems like it would feel out of place after the PR is merged