TruncateTransform.satisfies_order_of raises AttributeError for different widths

Apache Iceberg version

0.11.0 (latest release)

Please describe the bug 🐞

Description

Calling TruncateTransform.satisfies_order_of with two valid truncate transforms that have different widths raises an AttributeError instead of returning a boolean.

Same-width comparisons work because the method returns early when the transforms are equal.

Reproduction

from pyiceberg.transforms import TruncateTransform

TruncateTransform(5).satisfies_order_of(TruncateTransform(3))

On main at commit 48e710d20ceeeaa637d5aeae7746b787410859f8, this raises:

AttributeError: 'TruncateTransform' object has no attribute '_source_type'

The failure reproduces consistently. The same code is also present in the 0.11.1 release.

Expected behavior

The method should compare the truncate widths and return a boolean:

assert TruncateTransform(5).satisfies_order_of(TruncateTransform(3))
assert not TruncateTransform(3).satisfies_order_of(TruncateTransform(5))

This matches the current Apache Iceberg Java implementation:

https://github.com/apache/iceberg/blob/25654ab4b29c8b5b5c20fc427da01cb70d94ed14/api/src/main/java/org/apache/iceberg/transforms/Truncate.java#L130-L141

Cause

TruncateTransform.__init__ initializes _width but not _source_type. However, satisfies_order_of still accesses the source_type property backed by _source_type:

iceberg-python/pyiceberg/transforms.py

Lines 783 to 905 in 48e710d

    
           _source_type: IcebergType = PrivateAttr() 
        
           _width: PositiveInt = PrivateAttr() 
        
           def __init__(self, width: int, **data: Any): 
        
               super().__init__(root=f"truncate[{width}]", **data) 
        
               self._width = width 
        
           def can_transform(self, source: IcebergType) -> bool: 
        
               return isinstance(source, (IntegerType, LongType, StringType, BinaryType, DecimalType)) 
        
           def result_type(self, source: IcebergType) -> IcebergType: 
        
               return source 
        
           @property 
        
           def preserves_order(self) -> bool: 
        
               return True 
        
           @property 
        
           def source_type(self) -> IcebergType: 
        
               return self._source_type 
        
           def project(self, name: str, pred: BoundPredicate) -> UnboundPredicate | None: 
        
               field_type = pred.term.ref().field.field_type 
        
               if isinstance(pred.term, BoundTransform): 
        
                   return _project_transform_predicate(self, name, pred) 
        
               if isinstance(pred, BoundUnaryPredicate): 
        
                   return pred.as_unbound(Reference(name)) 
        
               elif isinstance(pred, BoundIn): 
        
                   return _set_apply_transform(name, pred, self.transform(field_type)) 
        
               elif isinstance(field_type, (IntegerType, LongType, DecimalType)):  # type: ignore 
        
                   if isinstance(pred, BoundLiteralPredicate): 
        
                       return _truncate_number(name, pred, self.transform(field_type)) 
        
               elif isinstance(field_type, (BinaryType, StringType)): 
        
                   if isinstance(pred, BoundLiteralPredicate): 
        
                       if isinstance(pred, BoundNotStartsWith): 
        
                           literal_width = len(pred.literal.value) 
        
                           if literal_width < self.width: 
        
                               return pred.as_unbound(name, pred.literal.value) 
        
                           elif literal_width == self.width: 
        
                               return NotEqualTo(name, pred.literal.value) 
        
                           else: 
        
                               return None 
        
                       else: 
        
                           return _truncate_array(name, pred, self.transform(field_type)) 
        
           def strict_project(self, name: str, pred: BoundPredicate) -> UnboundPredicate | None: 
        
               field_type = pred.term.ref().field.field_type 
        
               if isinstance(pred.term, BoundTransform): 
        
                   return _project_transform_predicate(self, name, pred) 
        
               if isinstance(pred, BoundUnaryPredicate): 
        
                   return pred.as_unbound(Reference(name)) 
        
               if isinstance(field_type, (IntegerType, LongType, DecimalType)): 
        
                   if isinstance(pred, BoundLiteralPredicate): 
        
                       return _truncate_number_strict(name, pred, self.transform(field_type)) 
        
                   elif isinstance(pred, BoundNotIn): 
        
                       return _set_apply_transform(name, pred, self.transform(field_type)) 
        
                   else: 
        
                       return None  # type: ignore 
        
               if isinstance(pred, BoundLiteralPredicate): 
        
                   if isinstance(pred, BoundStartsWith): 
        
                       literal_width = len(pred.literal.value) 
        
                       if literal_width < self.width: 
        
                           return pred.as_unbound(name, pred.literal.value) 
        
                       elif literal_width == self.width: 
        
                           return EqualTo(name, pred.literal.value) 
        
                       else: 
        
                           return None 
        
                   elif isinstance(pred, BoundNotStartsWith): 
        
                       literal_width = len(pred.literal.value) 
        
                       if literal_width < self.width: 
        
                           return pred.as_unbound(name, pred.literal.value) 
        
                       elif literal_width == self.width: 
        
                           return NotEqualTo(name, pred.literal.value) 
        
                       else: 
        
                           return pred.as_unbound(name, self.transform(field_type)(pred.literal.value)) 
        
                   else: 
        
                       # ProjectionUtil.truncateArrayStrict(name, pred, this); 
        
                       return _truncate_array_strict(name, pred, self.transform(field_type)) 
        
               elif isinstance(pred, BoundNotIn): 
        
                   return _set_apply_transform(name, pred, self.transform(field_type)) 
        
               else: 
        
                   return None  # type: ignore 
        
           @property 
        
           def width(self) -> int: 
        
               return self._width 
        
           def transform(self, source: IcebergType) -> Callable[[S | None], S | None]: 
        
               if isinstance(source, (IntegerType, LongType)): 
        
                   def truncate_func(v: Any) -> Any: 
        
                       return v - v % self._width 
        
               elif isinstance(source, (StringType, BinaryType)): 
        
                   def truncate_func(v: Any) -> Any: 
        
                       return v[0 : min(self._width, len(v))] 
        
               elif isinstance(source, DecimalType): 
        
                   def truncate_func(v: Any) -> Any: 
        
                       return truncate_decimal(v, self._width) 
        
               else: 
        
                   raise ValueError(f"Cannot truncate for type: {source}") 
        
               return lambda v: truncate_func(v) if v is not None else None 
        
           def satisfies_order_of(self, other: Transform[S, T]) -> bool: 
        
               if self == other: 
        
                   return True 
        
               elif ( 
        
                   isinstance(self.source_type, StringType) 
        
                   and isinstance(other, TruncateTransform) 
        
                   and isinstance(other.source_type, StringType) 
        
               ): 
        
                   return self.width >= other.width

The existing unit test only compares a transform with itself, so it returns before reaching the failing branch:

iceberg-python/tests/test_transforms.py

Lines 501 to 512 in 48e710d

    
           def test_truncate_method(type_var: PrimitiveType, value: Any, expected_human_str: str, expected: Any) -> None: 
        
               truncate_transform = TruncateTransform(1)  # type: ignore 
        
               assert str(truncate_transform) == str(eval(repr(truncate_transform))) 
        
               assert truncate_transform.can_transform(type_var) 
        
               assert truncate_transform.result_type(type_var) == type_var 
        
               assert truncate_transform.to_human_string(type_var, value) == expected_human_str 
        
               assert truncate_transform.transform(type_var)(value) == expected 
        
               assert truncate_transform.to_human_string(type_var, None) == "null" 
        
               assert truncate_transform.width == 1 
        
               assert truncate_transform.transform(type_var)(None) is None 
        
               assert truncate_transform.preserves_order 
        
               assert truncate_transform.satisfies_order_of(truncate_transform)

A focused fix could compare TruncateTransform widths directly, consistent with the Java implementation, and add regression cases for different widths.

I would be happy to contribute the fix and regression tests.

Willingness to contribute

I can contribute a fix for this bug independently
I would be willing to contribute a fix for this bug with guidance from the Iceberg community
I cannot contribute a fix for this bug at this time

Navigation Menu

Repository navigation

TruncateTransform.satisfies_order_of raises AttributeError for different widths #3680

Apache Iceberg version

Please describe the bug 🐞

Description

Reproduction

Expected behavior

Cause

Willingness to contribute

Metadata

Assignees

Labels

Type

Projects

Milestone

Relationships

Development

	_source_type: IcebergType = PrivateAttr()
	_width: PositiveInt = PrivateAttr()

	def __init__(self, width: int, **data: Any):
	super().__init__(root=f"truncate[{width}]", **data)
	self._width = width

	def can_transform(self, source: IcebergType) -> bool:
	return isinstance(source, (IntegerType, LongType, StringType, BinaryType, DecimalType))

	def result_type(self, source: IcebergType) -> IcebergType:
	return source

	@property
	def preserves_order(self) -> bool:
	return True

	@property
	def source_type(self) -> IcebergType:
	return self._source_type

	def project(self, name: str, pred: BoundPredicate) -> UnboundPredicate \| None:
	field_type = pred.term.ref().field.field_type

	if isinstance(pred.term, BoundTransform):
	return _project_transform_predicate(self, name, pred)

	if isinstance(pred, BoundUnaryPredicate):
	return pred.as_unbound(Reference(name))
	elif isinstance(pred, BoundIn):
	return _set_apply_transform(name, pred, self.transform(field_type))
	elif isinstance(field_type, (IntegerType, LongType, DecimalType)): # type: ignore
	if isinstance(pred, BoundLiteralPredicate):
	return _truncate_number(name, pred, self.transform(field_type))
	elif isinstance(field_type, (BinaryType, StringType)):
	if isinstance(pred, BoundLiteralPredicate):
	if isinstance(pred, BoundNotStartsWith):
	literal_width = len(pred.literal.value)
	if literal_width < self.width:
	return pred.as_unbound(name, pred.literal.value)
	elif literal_width == self.width:
	return NotEqualTo(name, pred.literal.value)
	else:
	return None
	else:
	return _truncate_array(name, pred, self.transform(field_type))

	def strict_project(self, name: str, pred: BoundPredicate) -> UnboundPredicate \| None:
	field_type = pred.term.ref().field.field_type

	if isinstance(pred.term, BoundTransform):
	return _project_transform_predicate(self, name, pred)

	if isinstance(pred, BoundUnaryPredicate):
	return pred.as_unbound(Reference(name))

	if isinstance(field_type, (IntegerType, LongType, DecimalType)):
	if isinstance(pred, BoundLiteralPredicate):
	return _truncate_number_strict(name, pred, self.transform(field_type))
	elif isinstance(pred, BoundNotIn):
	return _set_apply_transform(name, pred, self.transform(field_type))
	else:
	return None # type: ignore

	if isinstance(pred, BoundLiteralPredicate):
	if isinstance(pred, BoundStartsWith):
	literal_width = len(pred.literal.value)
	if literal_width < self.width:
	return pred.as_unbound(name, pred.literal.value)
	elif literal_width == self.width:
	return EqualTo(name, pred.literal.value)
	else:
	return None
	elif isinstance(pred, BoundNotStartsWith):
	literal_width = len(pred.literal.value)
	if literal_width < self.width:
	return pred.as_unbound(name, pred.literal.value)
	elif literal_width == self.width:
	return NotEqualTo(name, pred.literal.value)
	else:
	return pred.as_unbound(name, self.transform(field_type)(pred.literal.value))
	else:
	# ProjectionUtil.truncateArrayStrict(name, pred, this);
	return _truncate_array_strict(name, pred, self.transform(field_type))
	elif isinstance(pred, BoundNotIn):
	return _set_apply_transform(name, pred, self.transform(field_type))
	else:
	return None # type: ignore

	@property
	def width(self) -> int:
	return self._width

	def transform(self, source: IcebergType) -> Callable[[S \| None], S \| None]:
	if isinstance(source, (IntegerType, LongType)):

	def truncate_func(v: Any) -> Any:
	return v - v % self._width

	elif isinstance(source, (StringType, BinaryType)):

	def truncate_func(v: Any) -> Any:
	return v[0 : min(self._width, len(v))]

	elif isinstance(source, DecimalType):

	def truncate_func(v: Any) -> Any:
	return truncate_decimal(v, self._width)

	else:
	raise ValueError(f"Cannot truncate for type: {source}")

	return lambda v: truncate_func(v) if v is not None else None

	def satisfies_order_of(self, other: Transform[S, T]) -> bool:
	if self == other:
	return True
	elif (
	isinstance(self.source_type, StringType)
	and isinstance(other, TruncateTransform)
	and isinstance(other.source_type, StringType)
	):
	return self.width >= other.width

	def test_truncate_method(type_var: PrimitiveType, value: Any, expected_human_str: str, expected: Any) -> None:
	truncate_transform = TruncateTransform(1) # type: ignore
	assert str(truncate_transform) == str(eval(repr(truncate_transform)))
	assert truncate_transform.can_transform(type_var)
	assert truncate_transform.result_type(type_var) == type_var
	assert truncate_transform.to_human_string(type_var, value) == expected_human_str
	assert truncate_transform.transform(type_var)(value) == expected
	assert truncate_transform.to_human_string(type_var, None) == "null"
	assert truncate_transform.width == 1
	assert truncate_transform.transform(type_var)(None) is None
	assert truncate_transform.preserves_order
	assert truncate_transform.satisfies_order_of(truncate_transform)

TruncateTransform.satisfies_order_of raises AttributeError for different widths #3680

Description

Apache Iceberg version

Please describe the bug 🐞

Description

Reproduction

Expected behavior

Cause

Willingness to contribute

Activity

Metadata

Metadata

Assignees

Labels

Type

Projects

Milestone

Relationships

Development

Issue actions