From f42db79d4c86391efd444170d6ffd756083c7dd7 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Mon, 12 Oct 2026 14:35:27 +0800 Subject: [PATCH 1/8] Fix CASE expression caches leaking across TsBlocks --- .../it/db/it/IoTDBCaseWhenThenTableIT.java | 28 ++ .../process/FilterAndProjectOperator.java | 11 + ...AbstractCaseWhenThenColumnTransformer.java | 11 + .../process/FilterAndProjectOperatorTest.java | 430 ++++++++++++++++++ 4 files changed, 480 insertions(+) create mode 100644 iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperatorTest.java diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBCaseWhenThenTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBCaseWhenThenTableIT.java index 50053867636bd..ba56a57371ee8 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBCaseWhenThenTableIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBCaseWhenThenTableIT.java @@ -695,4 +695,32 @@ public void testThenWithBinarySameConstant() { String[] retArray = new String[] {"0,", "11,", "22,", "33,"}; tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE); } + + @Test + public void testCaseWhenReusesAggregationAcrossBlocks() { + prepareTableData( + new String[] { + "USE " + DATABASE, + "CREATE TABLE case_aggregation_cache (dev STRING TAG, c DOUBLE FIELD, q INT32 FIELD)", + "INSERT INTO case_aggregation_cache(time, dev, c, q) VALUES " + + "(0, 'd0', 1.0, 0), (0, 'd1', 1.0, 0), (3600000, 'd1', 1.0, 0)" + }); + + // The non-pushdown predicate and the different device/window shapes make the aggregation + // produce multiple blocks. CASE clears the shared aggregate before the next output uses it. + String suffix = " FROM case_aggregation_cache WHERE q % 2 < 1 GROUP BY dev, date_bin(1h, time)"; + for (String projection : + new String[] { + "CASE WHEN sum(c) > 0 THEN 1.0 END AS case_result, CAST(sum(c) AS DOUBLE) AS total", + "CASE WHEN sum(c) > 0 THEN 1.0 END AS case_result, sum(c) + 0 AS total", + "CASE WHEN sum(c) > 0 THEN CAST(sum(c) AS DOUBLE) / CAST(sum(c) AS DOUBLE) END " + + "AS case_result, CAST(sum(c) AS DOUBLE) AS total" + }) { + tableResultSetEqualTest( + "SELECT " + projection + suffix, + new String[] {"case_result", "total"}, + new String[] {"1.0,1.0,", "1.0,1.0,", "1.0,1.0,"}, + DATABASE); + } + } } diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperator.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperator.java index 5ab7e58a6962f..9742f1cea15a3 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperator.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperator.java @@ -152,6 +152,13 @@ public TsBlock next() throws Exception { private TsBlock getFilterTsBlock(TsBlock input) { final Column originTimeColumn = input.getTimeColumn(); final int positionCount = originTimeColumn.getPositionCount(); + // Short-circuit evaluation can reset cache reference counts when a shared expression is + // recomputed. Invalidate the whole DAG before feeding a new block, including intermediate + // expressions whose caches may still have unconsumed references from the previous block. + filterOutputTransformer.clearCache(); + for (ColumnTransformer columnTransformer : commonTransformerList) { + columnTransformer.clearCache(); + } // feed Filter ColumnTransformer, including TimeStampColumnTransformer and constant for (LeafColumnTransformer leafColumnTransformer : filterLeafColumnTransformerList) { leafColumnTransformer.initFromTsBlock(input); @@ -263,6 +270,10 @@ public static boolean satisfy(Column filterColumn, int rowIndex) { private TsBlock getTransformedTsBlock(TsBlock input) { final Column originTimeColumn = input.getTimeColumn(); final int positionCount = originTimeColumn.getPositionCount(); + // Clear all output DAGs before evaluating any output, preserving reuse within this block. + for (ColumnTransformer columnTransformer : projectOutputTransformerList) { + columnTransformer.clearCache(); + } // feed pre calculated data for (LeafColumnTransformer leafColumnTransformer : projectLeafColumnTransformerList) { leafColumnTransformer.initFromTsBlock(input); diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/AbstractCaseWhenThenColumnTransformer.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/AbstractCaseWhenThenColumnTransformer.java index 5603bde031d1a..dd65cc12f5dbf 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/AbstractCaseWhenThenColumnTransformer.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/AbstractCaseWhenThenColumnTransformer.java @@ -234,6 +234,17 @@ protected void checkType() { // do nothing } + @Override + public void clearCache() { + super.clearCache(); + for (Pair whenThenColumnTransformer : + whenThenTransformers) { + whenThenColumnTransformer.left.clearCache(); + whenThenColumnTransformer.right.clearCache(); + } + elseTransformer.clearCache(); + } + @Override public void close() { super.close(); diff --git a/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperatorTest.java b/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperatorTest.java new file mode 100644 index 0000000000000..e15152b0a43e5 --- /dev/null +++ b/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperatorTest.java @@ -0,0 +1,430 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.calc.execution.operator.process; + +import org.apache.iotdb.calc.execution.operator.CommonOperatorContext; +import org.apache.iotdb.calc.execution.operator.Operator; +import org.apache.iotdb.calc.plan.planner.memory.MemoryReservationManager; +import org.apache.iotdb.calc.transformation.dag.column.ColumnTransformer; +import org.apache.iotdb.calc.transformation.dag.column.FailFunctionColumnTransformer; +import org.apache.iotdb.calc.transformation.dag.column.TableCaseWhenThenColumnTransformer; +import org.apache.iotdb.calc.transformation.dag.column.binary.ArithmeticAdditionColumnTransformer; +import org.apache.iotdb.calc.transformation.dag.column.binary.CompareGreaterThanColumnTransformer; +import org.apache.iotdb.calc.transformation.dag.column.leaf.ConstantColumnTransformer; +import org.apache.iotdb.calc.transformation.dag.column.leaf.IdentityColumnTransformer; +import org.apache.iotdb.calc.transformation.dag.column.leaf.LeafColumnTransformer; +import org.apache.iotdb.calc.transformation.dag.column.leaf.NullColumnTransformer; +import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNodeId; + +import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.block.TsBlock; +import org.apache.tsfile.read.common.block.column.DoubleColumn; +import org.apache.tsfile.read.common.block.column.RunLengthEncodedColumn; +import org.apache.tsfile.read.common.block.column.TimeColumn; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.Optional; + +import static org.apache.tsfile.read.common.type.BooleanType.BOOLEAN; +import static org.apache.tsfile.read.common.type.DoubleType.DOUBLE; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +public class FilterAndProjectOperatorTest { + + @Test + public void testProjectionDoesNotReuseColumnsAcrossDifferentSizedBlocks() throws Exception { + Graph graph = new Graph(); + ColumnTransformer input = graph.input(0); + ColumnTransformer caseWhen = + graph.caseWhen(graph.greaterThan(input, 0), graph.constant(1), graph.nullValue()); + try (Operator operator = + project(graph, new TsBlock[] {block(1), block(-2, 3), block(4)}, caseWhen, input)) { + assertBlock(operator.next(), new Double[] {1.0}, new Double[] {1.0}); + assertBlock(operator.next(), new Double[] {null, 1.0}, new Double[] {-2.0, 3.0}); + assertBlock(operator.next(), new Double[] {1.0}, new Double[] {4.0}); + } + } + + @Test + public void testProjectionDoesNotReuseColumnsAcrossEqualSizedBlocks() throws Exception { + Graph graph = new Graph(); + ColumnTransformer input = graph.input(0); + ColumnTransformer caseWhen = + graph.caseWhen(graph.greaterThan(input, 0), graph.constant(1), graph.nullValue()); + try (Operator operator = + project(graph, new TsBlock[] {block(1, 2), block(-3, 4)}, caseWhen, input)) { + assertBlock(operator.next(), new Double[] {1.0, 1.0}, new Double[] {1.0, 2.0}); + assertBlock(operator.next(), new Double[] {null, 1.0}, new Double[] {-3.0, 4.0}); + } + } + + @Test + public void testProjectionHandlesEmptyBlocks() throws Exception { + Graph graph = new Graph(); + ColumnTransformer input = graph.input(0); + ColumnTransformer caseWhen = + graph.caseWhen(graph.greaterThan(input, 0), graph.constant(1), graph.nullValue()); + try (Operator operator = + project(graph, new TsBlock[] {block(1), block(), block(-2)}, caseWhen, input)) { + assertBlock(operator.next(), new Double[] {1.0}, new Double[] {1.0}); + assertBlock(operator.next(), new Double[0], new Double[0]); + assertBlock(operator.next(), new Double[] {null}, new Double[] {-2.0}); + } + } + + @Test + public void testProjectionClearsSharedIntermediateExpressions() throws Exception { + Graph graph = new Graph(); + ColumnTransformer shared = graph.add(graph.input(0), 0); + ColumnTransformer caseWhen = + graph.caseWhen(graph.greaterThan(shared, 0), graph.constant(1), graph.nullValue()); + try (Operator operator = + project(graph, new TsBlock[] {block(1), block(-2)}, caseWhen, graph.add(shared, 1))) { + assertBlock(operator.next(), new Double[] {1.0}, new Double[] {2.0}); + assertBlock(operator.next(), new Double[] {null}, new Double[] {-1.0}); + } + } + + @Test + public void testFilterDoesNotReuseColumnsAcrossBlocks() throws Exception { + Graph filter = new Graph(); + ColumnTransformer input = filter.input(0); + ColumnTransformer caseWhen = + filter.caseWhen(filter.greaterThan(input, 0), filter.constant(1), filter.constant(0)); + ColumnTransformer predicate = filter.greaterThan(filter.add(caseWhen, input), 0); + Graph projection = new Graph(); + try (Operator operator = + operator( + new TsBlock[] {block(1), block(-2, 3), block(-4)}, + filter, + predicate, + Collections.emptyList(), + projection, + projection.input(0))) { + assertBlock(operator.next(), new Double[] {1.0}); + assertBlock(operator.next(), new Double[] {3.0}); + assertBlock(operator.next(), new Double[0]); + } + } + + @Test + public void testFilterCommonExpressionsDoNotCrossBlocks() throws Exception { + for (boolean runLengthEncodedTime : new boolean[] {false, true}) { + Graph filter = new Graph(); + ColumnTransformer shared = filter.add(filter.input(0), 0); + ColumnTransformer caseWhen = + filter.caseWhen(filter.greaterThan(shared, 0), filter.constant(1), filter.constant(0)); + ColumnTransformer predicate = filter.greaterThan(caseWhen, 0); + Graph projection = new Graph(); + try (Operator operator = + operator( + new TsBlock[] { + block(runLengthEncodedTime, 1), + block(runLengthEncodedTime, -2, 3), + block(runLengthEncodedTime, 4, -5) + }, + filter, + predicate, + Collections.singletonList(shared), + projection, + projection.input(0), + projection.input(1))) { + assertBlock(operator.next(), new Double[] {1.0}, new Double[] {1.0}); + assertBlock(operator.next(), new Double[] {3.0}, new Double[] {3.0}); + assertBlock(operator.next(), new Double[] {4.0}, new Double[] {4.0}); + } + } + } + + @Test + public void testProjectionClearsCachedDescendantsOfNestedCase() throws Exception { + Graph graph = new Graph(); + ColumnTransformer shared = graph.add(graph.input(0), 0); + ColumnTransformer innerCase = + graph.caseWhen(graph.greaterThan(shared, 0), graph.constant(1), graph.constant(0)); + ColumnTransformer outerCase = + graph.caseWhen(graph.greaterThan(innerCase, 0), graph.constant(2), graph.constant(0)); + // Simulate an intermediate cache retained from the previous input block. Clearing only the + // outer CASE itself must not allow this cached value to reach the new block's WHEN condition. + shared.initializeColumnCache(doubleColumn(1)); + try (Operator operator = project(graph, new TsBlock[] {block(-2)}, outerCase)) { + assertBlock(operator.next(), new Double[] {0.0}); + } + } + + @Test + public void testThenExpressionIsRecomputedWithoutSelectionForLaterOutput() throws Exception { + Graph graph = new Graph(); + ColumnTransformer input = graph.input(0); + ColumnTransformer shared = graph.add(input, 1); + ColumnTransformer caseWhen = + graph.caseWhen(graph.greaterThan(input, 0), shared, graph.constant(0)); + try (Operator operator = + project(graph, new TsBlock[] {block(-2, 3), block(-4, 5)}, caseWhen, shared)) { + assertBlock(operator.next(), new Double[] {0.0, 4.0}, new Double[] {-1.0, 4.0}); + assertBlock(operator.next(), new Double[] {0.0, 6.0}, new Double[] {-3.0, 6.0}); + } + } + + @Test + public void testCaseStillSkipsUnusedFailBranch() throws Exception { + Graph graph = new Graph(); + ColumnTransformer caseWhen = + graph.caseWhen( + graph.greaterThan(graph.input(0), 0), + graph.constant(1), + new FailFunctionColumnTransformer(DOUBLE, "The unused ELSE branch was evaluated")); + try (Operator operator = project(graph, new TsBlock[] {block(1, 2), block(3)}, caseWhen)) { + assertBlock(operator.next(), new Double[] {1.0, 1.0}); + assertBlock(operator.next(), new Double[] {1.0}); + } + } + + @Test + public void testProjectionReusesSharedExpressionsWithinEachBlock() throws Exception { + Graph graph = new Graph(); + CountingIdentityColumnTransformer input = new CountingIdentityColumnTransformer(); + graph.leaves.add(input); + ColumnTransformer shared = graph.add(input, 0); + try (Operator operator = + project(graph, new TsBlock[] {block(1), block(2)}, shared, graph.add(shared, 1))) { + assertBlock(operator.next(), new Double[] {1.0}, new Double[] {2.0}); + assertEquals(1, input.evaluationCount); + assertBlock(operator.next(), new Double[] {2.0}, new Double[] {3.0}); + assertEquals(2, input.evaluationCount); + } + } + + private static FilterAndProjectOperator project( + Graph graph, TsBlock[] blocks, ColumnTransformer... outputs) { + return operator(blocks, null, null, Collections.emptyList(), graph, outputs); + } + + private static FilterAndProjectOperator operator( + TsBlock[] blocks, + Graph filter, + ColumnTransformer predicate, + List common, + Graph projection, + ColumnTransformer... outputs) { + List projectOutputs = new ArrayList<>(); + for (ColumnTransformer output : outputs) { + projectOutputs.add(reference(output)); + } + for (ColumnTransformer transformer : common) { + reference(transformer); + } + return new FilterAndProjectOperator( + new TestOperatorContext(), + new TsBlockSourceOperator(blocks), + Collections.nCopies(1 + common.size(), TSDataType.DOUBLE), + filter == null ? Collections.emptyList() : filter.leaves, + predicate == null ? null : reference(predicate), + common, + projection.leaves, + projectOutputs, + false, + predicate != null); + } + + private static ColumnTransformer reference(ColumnTransformer transformer) { + transformer.addReferenceCount(); + return transformer; + } + + private static DoubleColumn doubleColumn(double... values) { + return new DoubleColumn(values.length, Optional.empty(), values); + } + + private static TsBlock block(double... values) { + return block(false, values); + } + + private static TsBlock block(boolean runLengthEncodedTime, double... values) { + Column timeColumn = + runLengthEncodedTime + ? new RunLengthEncodedColumn(new TimeColumn(1, new long[] {0}), values.length) + : new TimeColumn(values.length, new long[values.length]); + return new TsBlock(timeColumn, doubleColumn(values)); + } + + private static void assertBlock(TsBlock actual, Double[]... expectedColumns) { + assertEquals(expectedColumns.length, actual.getValueColumnCount()); + assertEquals(expectedColumns[0].length, actual.getPositionCount()); + for (int columnIndex = 0; columnIndex < expectedColumns.length; columnIndex++) { + Column column = actual.getColumn(columnIndex); + Double[] expected = expectedColumns[columnIndex]; + assertEquals(expected.length, column.getPositionCount()); + for (int row = 0; row < expected.length; row++) { + if (expected[row] == null) { + assertTrue(column.isNull(row)); + } else { + assertFalse(column.isNull(row)); + assertEquals(expected[row], column.getDouble(row), 0.0); + } + } + } + } + + private static class Graph { + private final List leaves = new ArrayList<>(); + + private ColumnTransformer input(int index) { + return leaf(new IdentityColumnTransformer(DOUBLE, index)); + } + + private ColumnTransformer constant(double value) { + return leaf(new ConstantColumnTransformer(DOUBLE, doubleColumn(value))); + } + + private ColumnTransformer nullValue() { + return leaf(new NullColumnTransformer()); + } + + private ColumnTransformer leaf(LeafColumnTransformer transformer) { + leaves.add(transformer); + return transformer; + } + + private ColumnTransformer add(ColumnTransformer input, double value) { + return add(input, constant(value)); + } + + private ColumnTransformer add(ColumnTransformer left, ColumnTransformer right) { + return new ArithmeticAdditionColumnTransformer(DOUBLE, reference(left), reference(right)); + } + + private ColumnTransformer greaterThan(ColumnTransformer input, double value) { + return new CompareGreaterThanColumnTransformer( + BOOLEAN, reference(input), reference(constant(value))); + } + + private ColumnTransformer caseWhen( + ColumnTransformer when, ColumnTransformer then, ColumnTransformer otherwise) { + return new TableCaseWhenThenColumnTransformer( + DOUBLE, + Collections.singletonList(reference(when)), + Collections.singletonList(reference(then)), + reference(otherwise)); + } + } + + private static class CountingIdentityColumnTransformer extends IdentityColumnTransformer { + private int evaluationCount; + + private CountingIdentityColumnTransformer() { + super(DOUBLE, 0); + } + + @Override + protected void evaluate() { + evaluationCount++; + super.evaluate(); + } + } + + private static class TestOperatorContext extends CommonOperatorContext { + private TestOperatorContext() { + super(0, new PlanNodeId("test"), "test"); + } + + @Override + public MemoryReservationManager getMemoryReservationContext() { + return null; + } + + @Override + public int getFragmentId() { + return 0; + } + + @Override + public int getPipelineId() { + return 0; + } + + @Override + public long ramBytesUsed() { + return 0; + } + } + + private static class TsBlockSourceOperator implements Operator { + private final Iterator blocks; + private final CommonOperatorContext context = new TestOperatorContext(); + + private TsBlockSourceOperator(TsBlock[] blocks) { + this.blocks = Arrays.asList(blocks).iterator(); + } + + @Override + public CommonOperatorContext getOperatorContext() { + return context; + } + + @Override + public TsBlock next() { + return blocks.next(); + } + + @Override + public boolean hasNext() { + return blocks.hasNext(); + } + + @Override + public boolean isFinished() { + return !hasNext(); + } + + @Override + public void close() { + // No resources to release. + } + + @Override + public long calculateMaxPeekMemory() { + return 0; + } + + @Override + public long calculateMaxReturnSize() { + return 0; + } + + @Override + public long calculateRetainedSizeAfterCallingNext() { + return 0; + } + + @Override + public long ramBytesUsed() { + return 0; + } + } +} From b863e7b6c20664e086eb574c2f995b8b80c2a102 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Mon, 12 Oct 2026 18:27:42 +0800 Subject: [PATCH 2/8] Separate block-boundary cache clearing from CASE cleanup --- .../it/db/it/IoTDBCaseWhenThenTableIT.java | 22 +++++++++++++++++++ .../process/FilterAndProjectOperator.java | 6 ++--- ...AbstractCaseWhenThenColumnTransformer.java | 10 ++++----- .../dag/column/ColumnTransformer.java | 9 ++++++++ .../binary/BinaryColumnTransformer.java | 7 ++++++ .../multi/MappableUDFColumnTransformer.java | 8 +++++++ .../column/multi/MultiColumnTransformer.java | 8 +++++++ .../ternary/TernaryColumnTransformer.java | 8 +++++++ .../column/unary/UnaryColumnTransformer.java | 6 +++++ .../process/FilterAndProjectOperatorTest.java | 21 ++++++++++++++++++ 10 files changed, 97 insertions(+), 8 deletions(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBCaseWhenThenTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBCaseWhenThenTableIT.java index ba56a57371ee8..169ebd3084ae3 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBCaseWhenThenTableIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBCaseWhenThenTableIT.java @@ -723,4 +723,26 @@ public void testCaseWhenReusesAggregationAcrossBlocks() { DATABASE); } } + + @Test + public void testReusingCasePreservesPendingSiblingColumn() { + prepareTableData( + new String[] { + "USE " + DATABASE, + "CREATE TABLE case_sibling_cache (dev STRING TAG, x DOUBLE FIELD)", + "INSERT INTO case_sibling_cache(time, dev, x) VALUES " + + "(0, 'd0', -3.0), (1, 'd0', 0.0), (2, 'd0', 2.0)" + }); + + String sharedCase = "CASE WHEN x > 0 THEN 1.0 ELSE 0.0 END"; + tableResultSetEqualTest( + "SELECT " + + sharedCase + + " AS a, x + CASE WHEN (" + + sharedCase + + ") > 0 THEN 2.0 ELSE 3.0 END AS b FROM case_sibling_cache ORDER BY time", + new String[] {"a", "b"}, + new String[] {"0.0,0.0,", "0.0,3.0,", "1.0,4.0,"}, + DATABASE); + } } diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperator.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperator.java index 9742f1cea15a3..29b51ac5cb663 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperator.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperator.java @@ -155,9 +155,9 @@ private TsBlock getFilterTsBlock(TsBlock input) { // Short-circuit evaluation can reset cache reference counts when a shared expression is // recomputed. Invalidate the whole DAG before feeding a new block, including intermediate // expressions whose caches may still have unconsumed references from the previous block. - filterOutputTransformer.clearCache(); + filterOutputTransformer.clearCacheForNewInput(); for (ColumnTransformer columnTransformer : commonTransformerList) { - columnTransformer.clearCache(); + columnTransformer.clearCacheForNewInput(); } // feed Filter ColumnTransformer, including TimeStampColumnTransformer and constant for (LeafColumnTransformer leafColumnTransformer : filterLeafColumnTransformerList) { @@ -272,7 +272,7 @@ private TsBlock getTransformedTsBlock(TsBlock input) { final int positionCount = originTimeColumn.getPositionCount(); // Clear all output DAGs before evaluating any output, preserving reuse within this block. for (ColumnTransformer columnTransformer : projectOutputTransformerList) { - columnTransformer.clearCache(); + columnTransformer.clearCacheForNewInput(); } // feed pre calculated data for (LeafColumnTransformer leafColumnTransformer : projectLeafColumnTransformerList) { diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/AbstractCaseWhenThenColumnTransformer.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/AbstractCaseWhenThenColumnTransformer.java index dd65cc12f5dbf..bcf4bdf7722cb 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/AbstractCaseWhenThenColumnTransformer.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/AbstractCaseWhenThenColumnTransformer.java @@ -235,14 +235,14 @@ protected void checkType() { } @Override - public void clearCache() { - super.clearCache(); + public void clearCacheForNewInput() { + super.clearCacheForNewInput(); for (Pair whenThenColumnTransformer : whenThenTransformers) { - whenThenColumnTransformer.left.clearCache(); - whenThenColumnTransformer.right.clearCache(); + whenThenColumnTransformer.left.clearCacheForNewInput(); + whenThenColumnTransformer.right.clearCacheForNewInput(); } - elseTransformer.clearCache(); + elseTransformer.clearCacheForNewInput(); } @Override diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/ColumnTransformer.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/ColumnTransformer.java index d10d728cd2db5..11407afbdd227 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/ColumnTransformer.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/ColumnTransformer.java @@ -115,4 +115,13 @@ public void close() { public void clearCache() { this.columnCache.clear(); } + + /** + * Clears cached columns throughout this DAG before initializing a new input block. Unlike {@link + * #clearCache()}, this also visits CASE descendants. Only call this before evaluating any + * expressions for the new block, when no sibling expression can still need a cached column. + */ + public void clearCacheForNewInput() { + this.columnCache.clear(); + } } diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/binary/BinaryColumnTransformer.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/binary/BinaryColumnTransformer.java index 5403e9f995f13..d494f652c8079 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/binary/BinaryColumnTransformer.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/binary/BinaryColumnTransformer.java @@ -96,6 +96,13 @@ public void clearCache() { this.rightTransformer.clearCache(); } + @Override + public void clearCacheForNewInput() { + super.clearCacheForNewInput(); + leftTransformer.clearCacheForNewInput(); + rightTransformer.clearCacheForNewInput(); + } + @Override public void close() { super.close(); diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/MappableUDFColumnTransformer.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/MappableUDFColumnTransformer.java index f619f5692119c..75ed45ac64278 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/MappableUDFColumnTransformer.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/MappableUDFColumnTransformer.java @@ -100,4 +100,12 @@ public void clearCache() { columnTransformer.clearCache(); } } + + @Override + public void clearCacheForNewInput() { + super.clearCacheForNewInput(); + for (ColumnTransformer columnTransformer : inputColumnTransformers) { + columnTransformer.clearCacheForNewInput(); + } + } } diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/MultiColumnTransformer.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/MultiColumnTransformer.java index 09cb1530bc475..0cfecdd87e5f3 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/MultiColumnTransformer.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/MultiColumnTransformer.java @@ -97,6 +97,14 @@ public void clearCache() { } } + @Override + public void clearCacheForNewInput() { + super.clearCacheForNewInput(); + for (ColumnTransformer columnTransformer : columnTransformerList) { + columnTransformer.clearCacheForNewInput(); + } + } + @Override public void close() { super.close(); diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/ternary/TernaryColumnTransformer.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/ternary/TernaryColumnTransformer.java index 51648f797e3f7..f68cc612da875 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/ternary/TernaryColumnTransformer.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/ternary/TernaryColumnTransformer.java @@ -111,6 +111,14 @@ public void clearCache() { thirdColumnTransformer.clearCache(); } + @Override + public void clearCacheForNewInput() { + super.clearCacheForNewInput(); + firstColumnTransformer.clearCacheForNewInput(); + secondColumnTransformer.clearCacheForNewInput(); + thirdColumnTransformer.clearCacheForNewInput(); + } + @Override public void close() { super.close(); diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/unary/UnaryColumnTransformer.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/unary/UnaryColumnTransformer.java index 2a88aea340a95..1a8943357cd15 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/unary/UnaryColumnTransformer.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/unary/UnaryColumnTransformer.java @@ -73,6 +73,12 @@ public void clearCache() { childColumnTransformer.clearCache(); } + @Override + public void clearCacheForNewInput() { + super.clearCacheForNewInput(); + childColumnTransformer.clearCacheForNewInput(); + } + @Override public void close() { super.close(); diff --git a/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperatorTest.java b/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperatorTest.java index e15152b0a43e5..92d4931ea5674 100644 --- a/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperatorTest.java +++ b/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperatorTest.java @@ -220,6 +220,27 @@ public void testProjectionReusesSharedExpressionsWithinEachBlock() throws Except } } + @Test + public void testReusingCasePreservesPendingSiblingColumn() throws Exception { + Graph graph = new Graph(); + ColumnTransformer input = graph.input(0); + ColumnTransformer sharedCase = + graph.caseWhen(graph.greaterThan(input, 0), graph.constant(1), graph.constant(0)); + ColumnTransformer nestedCase = + graph.caseWhen(graph.greaterThan(sharedCase, 0), graph.constant(2), graph.constant(3)); + // The first output leaves sharedCase cached. In the second output, input is evaluated before + // nestedCase reuses sharedCase. Its cleanup must preserve input until the addition reads it. + try (Operator operator = + project( + graph, + new TsBlock[] {block(-3, 0, 2), block(4, -1)}, + sharedCase, + graph.add(input, nestedCase))) { + assertBlock(operator.next(), new Double[] {0.0, 0.0, 1.0}, new Double[] {0.0, 3.0, 4.0}); + assertBlock(operator.next(), new Double[] {1.0, 0.0}, new Double[] {6.0, 2.0}); + } + } + private static FilterAndProjectOperator project( Graph graph, TsBlock[] blocks, ColumnTransformer... outputs) { return operator(blocks, null, null, Collections.emptyList(), graph, outputs); From dc2ff7dbcfca907944ccf71a1d85bc41aa11fd70 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Tue, 13 Oct 2026 10:19:13 +0800 Subject: [PATCH 3/8] Deduplicate transformer cache clearing across shared DAGs --- .../process/FilterAndProjectOperator.java | 13 +++-- ...AbstractCaseWhenThenColumnTransformer.java | 10 ++-- .../dag/column/ColumnTransformer.java | 16 +++++- .../binary/BinaryColumnTransformer.java | 9 ++-- .../multi/MappableUDFColumnTransformer.java | 7 +-- .../column/multi/MultiColumnTransformer.java | 6 +-- .../ternary/TernaryColumnTransformer.java | 11 ++-- .../column/unary/UnaryColumnTransformer.java | 7 +-- .../process/FilterAndProjectOperatorTest.java | 51 +++++++++++++++++++ 9 files changed, 101 insertions(+), 29 deletions(-) diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperator.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperator.java index 29b51ac5cb663..d764114496767 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperator.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperator.java @@ -47,8 +47,11 @@ import org.apache.tsfile.utils.RamUsageEstimator; import java.util.ArrayList; +import java.util.Collections; +import java.util.IdentityHashMap; import java.util.List; import java.util.OptionalInt; +import java.util.Set; public class FilterAndProjectOperator implements ProcessOperator { @@ -155,9 +158,10 @@ private TsBlock getFilterTsBlock(TsBlock input) { // Short-circuit evaluation can reset cache reference counts when a shared expression is // recomputed. Invalidate the whole DAG before feeding a new block, including intermediate // expressions whose caches may still have unconsumed references from the previous block. - filterOutputTransformer.clearCacheForNewInput(); + Set visited = Collections.newSetFromMap(new IdentityHashMap<>()); + filterOutputTransformer.clearCacheForNewInput(visited); for (ColumnTransformer columnTransformer : commonTransformerList) { - columnTransformer.clearCacheForNewInput(); + columnTransformer.clearCacheForNewInput(visited); } // feed Filter ColumnTransformer, including TimeStampColumnTransformer and constant for (LeafColumnTransformer leafColumnTransformer : filterLeafColumnTransformerList) { @@ -270,9 +274,10 @@ public static boolean satisfy(Column filterColumn, int rowIndex) { private TsBlock getTransformedTsBlock(TsBlock input) { final Column originTimeColumn = input.getTimeColumn(); final int positionCount = originTimeColumn.getPositionCount(); - // Clear all output DAGs before evaluating any output, preserving reuse within this block. + // Clear all output DAGs before evaluating any output, visiting shared subexpressions only once. + Set visited = Collections.newSetFromMap(new IdentityHashMap<>()); for (ColumnTransformer columnTransformer : projectOutputTransformerList) { - columnTransformer.clearCacheForNewInput(); + columnTransformer.clearCacheForNewInput(visited); } // feed pre calculated data for (LeafColumnTransformer leafColumnTransformer : projectLeafColumnTransformerList) { diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/AbstractCaseWhenThenColumnTransformer.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/AbstractCaseWhenThenColumnTransformer.java index bcf4bdf7722cb..bc083f65b3b45 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/AbstractCaseWhenThenColumnTransformer.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/AbstractCaseWhenThenColumnTransformer.java @@ -30,6 +30,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.Set; public abstract class AbstractCaseWhenThenColumnTransformer extends ColumnTransformer { @@ -235,14 +236,13 @@ protected void checkType() { } @Override - public void clearCacheForNewInput() { - super.clearCacheForNewInput(); + protected void clearChildrenCacheForNewInput(Set visited) { for (Pair whenThenColumnTransformer : whenThenTransformers) { - whenThenColumnTransformer.left.clearCacheForNewInput(); - whenThenColumnTransformer.right.clearCacheForNewInput(); + whenThenColumnTransformer.left.clearCacheForNewInput(visited); + whenThenColumnTransformer.right.clearCacheForNewInput(visited); } - elseTransformer.clearCacheForNewInput(); + elseTransformer.clearCacheForNewInput(visited); } @Override diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/ColumnTransformer.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/ColumnTransformer.java index 11407afbdd227..8e12b18e720c0 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/ColumnTransformer.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/ColumnTransformer.java @@ -25,6 +25,8 @@ import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.read.common.type.TypeEnum; +import java.util.Set; + import static org.apache.iotdb.calc.plan.relational.metadata.CommonMetadataUtils.isNumericType; public abstract class ColumnTransformer { @@ -119,9 +121,19 @@ public void clearCache() { /** * Clears cached columns throughout this DAG before initializing a new input block. Unlike {@link * #clearCache()}, this also visits CASE descendants. Only call this before evaluating any - * expressions for the new block, when no sibling expression can still need a cached column. + * expressions for the new block, when no sibling expression can still need a cached column. All + * roots in the same evaluation phase must share an identity-based visited set so shared + * subexpressions are cleared only once. */ - public void clearCacheForNewInput() { + public final void clearCacheForNewInput(Set visited) { + if (!visited.add(this)) { + return; + } this.columnCache.clear(); + clearChildrenCacheForNewInput(visited); + } + + protected void clearChildrenCacheForNewInput(Set visited) { + // Leaf transformers have no children. } } diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/binary/BinaryColumnTransformer.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/binary/BinaryColumnTransformer.java index d494f652c8079..1483e96416e67 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/binary/BinaryColumnTransformer.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/binary/BinaryColumnTransformer.java @@ -25,6 +25,8 @@ import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.read.common.type.Type; +import java.util.Set; + public abstract class BinaryColumnTransformer extends ColumnTransformer { protected final ColumnTransformer leftTransformer; @@ -97,10 +99,9 @@ public void clearCache() { } @Override - public void clearCacheForNewInput() { - super.clearCacheForNewInput(); - leftTransformer.clearCacheForNewInput(); - rightTransformer.clearCacheForNewInput(); + protected void clearChildrenCacheForNewInput(Set visited) { + leftTransformer.clearCacheForNewInput(visited); + rightTransformer.clearCacheForNewInput(visited); } @Override diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/MappableUDFColumnTransformer.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/MappableUDFColumnTransformer.java index 75ed45ac64278..0d7576025c5e8 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/MappableUDFColumnTransformer.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/MappableUDFColumnTransformer.java @@ -26,6 +26,8 @@ import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.read.common.type.Type; +import java.util.Set; + public class MappableUDFColumnTransformer extends ColumnTransformer { private final ColumnTransformer[] inputColumnTransformers; @@ -102,10 +104,9 @@ public void clearCache() { } @Override - public void clearCacheForNewInput() { - super.clearCacheForNewInput(); + protected void clearChildrenCacheForNewInput(Set visited) { for (ColumnTransformer columnTransformer : inputColumnTransformers) { - columnTransformer.clearCacheForNewInput(); + columnTransformer.clearCacheForNewInput(visited); } } } diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/MultiColumnTransformer.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/MultiColumnTransformer.java index 0cfecdd87e5f3..5d8be08896d91 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/MultiColumnTransformer.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/MultiColumnTransformer.java @@ -27,6 +27,7 @@ import java.util.ArrayList; import java.util.List; +import java.util.Set; import java.util.stream.Collectors; public abstract class MultiColumnTransformer extends ColumnTransformer { @@ -98,10 +99,9 @@ public void clearCache() { } @Override - public void clearCacheForNewInput() { - super.clearCacheForNewInput(); + protected void clearChildrenCacheForNewInput(Set visited) { for (ColumnTransformer columnTransformer : columnTransformerList) { - columnTransformer.clearCacheForNewInput(); + columnTransformer.clearCacheForNewInput(visited); } } diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/ternary/TernaryColumnTransformer.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/ternary/TernaryColumnTransformer.java index f68cc612da875..1c8dfce914a8d 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/ternary/TernaryColumnTransformer.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/ternary/TernaryColumnTransformer.java @@ -25,6 +25,8 @@ import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.read.common.type.Type; +import java.util.Set; + public abstract class TernaryColumnTransformer extends ColumnTransformer { protected ColumnTransformer firstColumnTransformer; @@ -112,11 +114,10 @@ public void clearCache() { } @Override - public void clearCacheForNewInput() { - super.clearCacheForNewInput(); - firstColumnTransformer.clearCacheForNewInput(); - secondColumnTransformer.clearCacheForNewInput(); - thirdColumnTransformer.clearCacheForNewInput(); + protected void clearChildrenCacheForNewInput(Set visited) { + firstColumnTransformer.clearCacheForNewInput(visited); + secondColumnTransformer.clearCacheForNewInput(visited); + thirdColumnTransformer.clearCacheForNewInput(visited); } @Override diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/unary/UnaryColumnTransformer.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/unary/UnaryColumnTransformer.java index 1a8943357cd15..3e28d3cfe874a 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/unary/UnaryColumnTransformer.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/unary/UnaryColumnTransformer.java @@ -25,6 +25,8 @@ import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.read.common.type.Type; +import java.util.Set; + public abstract class UnaryColumnTransformer extends ColumnTransformer { protected ColumnTransformer childColumnTransformer; @@ -74,9 +76,8 @@ public void clearCache() { } @Override - public void clearCacheForNewInput() { - super.clearCacheForNewInput(); - childColumnTransformer.clearCacheForNewInput(); + protected void clearChildrenCacheForNewInput(Set visited) { + childColumnTransformer.clearCacheForNewInput(visited); } @Override diff --git a/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperatorTest.java b/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperatorTest.java index 92d4931ea5674..d50587fdea757 100644 --- a/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperatorTest.java +++ b/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperatorTest.java @@ -47,6 +47,7 @@ import java.util.Iterator; import java.util.List; import java.util.Optional; +import java.util.Set; import static org.apache.tsfile.read.common.type.BooleanType.BOOLEAN; import static org.apache.tsfile.read.common.type.DoubleType.DOUBLE; @@ -220,6 +221,50 @@ public void testProjectionReusesSharedExpressionsWithinEachBlock() throws Except } } + @Test + public void testProjectionClearsSharedDagOnlyOncePerBlock() throws Exception { + Graph graph = new Graph(); + CountingIdentityColumnTransformer input = new CountingIdentityColumnTransformer(); + graph.leaves.add(input); + ColumnTransformer shared = input; + // Each level references the same child twice. A traversal without deduplication would visit + // the input 4096 times per root, although this DAG contains only 13 distinct transformers. + for (int i = 0; i < 12; i++) { + shared = graph.add(shared, shared); + } + try (Operator operator = + project(graph, new TsBlock[] {block(1), block(2)}, shared, graph.add(shared, 1))) { + assertBlock(operator.next(), new Double[] {4096.0}, new Double[] {4097.0}); + assertEquals(1, input.cacheClearCount); + assertBlock(operator.next(), new Double[] {8192.0}, new Double[] {8193.0}); + assertEquals(2, input.cacheClearCount); + } + } + + @Test + public void testFilterAndCommonExpressionsShareCacheClearingTraversal() throws Exception { + Graph filter = new Graph(); + CountingIdentityColumnTransformer input = new CountingIdentityColumnTransformer(); + filter.leaves.add(input); + ColumnTransformer shared = filter.add(input, 1); + ColumnTransformer predicate = filter.greaterThan(shared, 0); + Graph projection = new Graph(); + try (Operator operator = + operator( + new TsBlock[] {block(1), block(-2, 3)}, + filter, + predicate, + Collections.singletonList(shared), + projection, + projection.input(0), + projection.input(1))) { + assertBlock(operator.next(), new Double[] {1.0}, new Double[] {2.0}); + assertEquals(1, input.cacheClearCount); + assertBlock(operator.next(), new Double[] {3.0}, new Double[] {4.0}); + assertEquals(2, input.cacheClearCount); + } + } + @Test public void testReusingCasePreservesPendingSiblingColumn() throws Exception { Graph graph = new Graph(); @@ -357,6 +402,7 @@ private ColumnTransformer caseWhen( private static class CountingIdentityColumnTransformer extends IdentityColumnTransformer { private int evaluationCount; + private int cacheClearCount; private CountingIdentityColumnTransformer() { super(DOUBLE, 0); @@ -367,6 +413,11 @@ protected void evaluate() { evaluationCount++; super.evaluate(); } + + @Override + protected void clearChildrenCacheForNewInput(Set visited) { + cacheClearCount++; + } } private static class TestOperatorContext extends CommonOperatorContext { From 3d10a44e2f2af096b0b4e18a707ef5c37f6f9acd Mon Sep 17 00:00:00 2001 From: ColinLee Date: Tue, 13 Oct 2026 10:52:10 +0800 Subject: [PATCH 4/8] Clear CASE caches for schema batches and unify DAG traversal --- .../it/schema/IoTDBCaseDeviceCacheIT.java | 136 ++++++++ .../process/FilterAndProjectOperator.java | 4 +- ...AbstractCaseWhenThenColumnTransformer.java | 16 +- .../dag/column/ColumnTransformer.java | 15 +- .../column/FailFunctionColumnTransformer.java | 7 + .../binary/BinaryColumnTransformer.java | 15 +- .../column/leaf/LeafColumnTransformer.java | 7 + .../multi/MappableUDFColumnTransformer.java | 14 +- .../column/multi/MultiColumnTransformer.java | 16 +- .../ternary/TernaryColumnTransformer.java | 18 +- .../column/unary/UnaryColumnTransformer.java | 12 +- .../process/FilterAndProjectOperatorTest.java | 39 ++- .../schema/source/DeviceAttributeUpdater.java | 19 +- .../schema/source/DevicePredicateHandler.java | 9 + .../source/DevicePredicateHandlerTest.java | 323 ++++++++++++++++++ 15 files changed, 578 insertions(+), 72 deletions(-) create mode 100644 integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBCaseDeviceCacheIT.java create mode 100644 iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DevicePredicateHandlerTest.java diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBCaseDeviceCacheIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBCaseDeviceCacheIT.java new file mode 100644 index 0000000000000..3ef2464181106 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBCaseDeviceCacheIT.java @@ -0,0 +1,136 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.schema; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; +import org.apache.iotdb.itbase.env.BaseEnv; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; + +import static org.apache.iotdb.db.it.utils.TestUtils.assertResultSetEqual; +import static org.apache.iotdb.db.it.utils.TestUtils.prepareTableData; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBCaseDeviceCacheIT { + private static final String DATABASE = "case_device_cache"; + private static final String CASE_RESULT = "CASE WHEN src = 'match' THEN 'match' ELSE 'other' END"; + + @BeforeClass + public static void setUp() throws Exception { + // Keep every device in the same schema region and force a new batch for each device. + EnvFactory.getEnv().getConfig().getCommonConfig().setDefaultSchemaRegionGroupNumPerDatabase(1); + EnvFactory.getEnv().getConfig().getCommonConfig().setMaxTsBlockLineNumber(1); + EnvFactory.getEnv().initClusterEnvironment(); + prepareTableData(new String[] {"CREATE DATABASE " + DATABASE}); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void testDevicePredicatesAcrossBatches() throws Exception { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + createTable(statement, "predicate_cache"); + String predicate = CASE_RESULT + " = src"; + assertResultSetEqual( + statement.executeQuery("SHOW DEVICES FROM predicate_cache WHERE " + predicate), + "id,src,a,b,", + new HashSet<>( + Arrays.asList( + "d0,match,null,null,", + "d1,match,null,null,", + "d2,other,null,null,", + "d3,other,null,null,"))); + assertResultSetEqual( + statement.executeQuery("COUNT DEVICES FROM predicate_cache WHERE " + predicate), + "count(devices),", + Collections.singleton("4,")); + } + } + + @Test + public void testAttributeProjectionAcrossBatches() throws Exception { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + createTable(statement, "projection_cache"); + statement.execute("UPDATE projection_cache SET a = " + CASE_RESULT + ", b = src"); + assertResultSetEqual( + statement.executeQuery("SHOW DEVICES FROM projection_cache"), + "id,src,a,b,", + new HashSet<>( + Arrays.asList( + "d0,match,match,match,", + "d1,match,match,match,", + "d2,other,other,other,", + "d3,other,other,other,"))); + } + } + + @Test + public void testAttributeCommonExpressionClearedByCase() throws Exception { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + createTable(statement, "common_cache"); + statement.execute( + "UPDATE common_cache SET a = src " + + "WHERE CASE WHEN src = 'match' THEN true ELSE false END"); + assertResultSetEqual( + statement.executeQuery("SHOW DEVICES FROM common_cache"), + "id,src,a,b,", + new HashSet<>( + Arrays.asList( + "d0,match,match,null,", + "d1,match,match,null,", + "d2,other,null,null,", + "d3,other,null,null,"))); + } + } + + private static void createTable(Statement statement, String table) throws SQLException { + statement.execute("USE " + DATABASE); + statement.execute( + "CREATE TABLE " + + table + + " (id STRING TAG, src STRING ATTRIBUTE, a STRING ATTRIBUTE, b STRING ATTRIBUTE, v DOUBLE FIELD)"); + statement.execute( + "INSERT INTO " + + table + + "(time, id, src, v) VALUES (1, 'd0', 'match', 1), (2, 'd1', 'match', 1), " + + "(3, 'd2', 'other', 1), (4, 'd3', 'other', 1)"); + } +} diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperator.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperator.java index d764114496767..7598a0c2196b8 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperator.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperator.java @@ -183,8 +183,8 @@ private TsBlock getFilterTsBlock(TsBlock input) { if (!hasNonMappableUDF) { // get result of calculated common sub expressions for (ColumnTransformer columnTransformer : commonTransformerList) { - // CASE WHEN clause would clear all its cache - // evaluate again to acquire cache + // CASE evaluation can clear a common expression after the filter used it. Recompute it + // before materializing the column for projection within this same input block. columnTransformer.tryEvaluate(); resultColumns.add(columnTransformer.getColumn()); } diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/AbstractCaseWhenThenColumnTransformer.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/AbstractCaseWhenThenColumnTransformer.java index bc083f65b3b45..e110b7331b120 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/AbstractCaseWhenThenColumnTransformer.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/AbstractCaseWhenThenColumnTransformer.java @@ -30,7 +30,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; -import java.util.Set; +import java.util.function.Consumer; public abstract class AbstractCaseWhenThenColumnTransformer extends ColumnTransformer { @@ -236,13 +236,19 @@ protected void checkType() { } @Override - protected void clearChildrenCacheForNewInput(Set visited) { + protected boolean shouldClearChildrenCache() { + // Clearing a cached CASE during evaluation must preserve pending sibling columns. + return false; + } + + @Override + protected void forEachChild(Consumer action) { for (Pair whenThenColumnTransformer : whenThenTransformers) { - whenThenColumnTransformer.left.clearCacheForNewInput(visited); - whenThenColumnTransformer.right.clearCacheForNewInput(visited); + action.accept(whenThenColumnTransformer.left); + action.accept(whenThenColumnTransformer.right); } - elseTransformer.clearCacheForNewInput(visited); + action.accept(elseTransformer); } @Override diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/ColumnTransformer.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/ColumnTransformer.java index 8e12b18e720c0..e8dcbee6fb0fb 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/ColumnTransformer.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/ColumnTransformer.java @@ -26,6 +26,7 @@ import org.apache.tsfile.read.common.type.TypeEnum; import java.util.Set; +import java.util.function.Consumer; import static org.apache.iotdb.calc.plan.relational.metadata.CommonMetadataUtils.isNumericType; @@ -114,8 +115,11 @@ public void close() { // do nothing } - public void clearCache() { + public final void clearCache() { this.columnCache.clear(); + if (shouldClearChildrenCache()) { + forEachChild(ColumnTransformer::clearCache); + } } /** @@ -130,10 +134,13 @@ public final void clearCacheForNewInput(Set visited) { return; } this.columnCache.clear(); - clearChildrenCacheForNewInput(visited); + forEachChild(child -> child.clearCacheForNewInput(visited)); } - protected void clearChildrenCacheForNewInput(Set visited) { - // Leaf transformers have no children. + protected boolean shouldClearChildrenCache() { + return true; } + + /** Defines the child edges used by both evaluation-time and new-input cache clearing. */ + protected abstract void forEachChild(Consumer action); } diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/FailFunctionColumnTransformer.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/FailFunctionColumnTransformer.java index 351c4396c0fc3..07b484deff0d4 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/FailFunctionColumnTransformer.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/FailFunctionColumnTransformer.java @@ -24,6 +24,8 @@ import org.apache.tsfile.read.common.block.column.NullColumn; import org.apache.tsfile.read.common.type.Type; +import java.util.function.Consumer; + /** * when evaluate of FailFunctionColumnTransformer is called, throw Exception to inform the user with * specified errorMsg @@ -59,4 +61,9 @@ public void evaluateWithSelection(boolean[] selection) { protected void checkType() { // do nothing } + + @Override + protected void forEachChild(Consumer action) { + // The fail function has no children. + } } diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/binary/BinaryColumnTransformer.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/binary/BinaryColumnTransformer.java index 1483e96416e67..d2ca5270dee0f 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/binary/BinaryColumnTransformer.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/binary/BinaryColumnTransformer.java @@ -25,7 +25,7 @@ import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.read.common.type.Type; -import java.util.Set; +import java.util.function.Consumer; public abstract class BinaryColumnTransformer extends ColumnTransformer { @@ -92,16 +92,9 @@ public ColumnTransformer getRightTransformer() { } @Override - public void clearCache() { - super.clearCache(); - this.leftTransformer.clearCache(); - this.rightTransformer.clearCache(); - } - - @Override - protected void clearChildrenCacheForNewInput(Set visited) { - leftTransformer.clearCacheForNewInput(visited); - rightTransformer.clearCacheForNewInput(visited); + protected void forEachChild(Consumer action) { + action.accept(leftTransformer); + action.accept(rightTransformer); } @Override diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/leaf/LeafColumnTransformer.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/leaf/LeafColumnTransformer.java index e9f69657a5405..ce16518ce245d 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/leaf/LeafColumnTransformer.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/leaf/LeafColumnTransformer.java @@ -24,6 +24,8 @@ import org.apache.tsfile.read.common.block.TsBlock; import org.apache.tsfile.read.common.type.Type; +import java.util.function.Consumer; + public abstract class LeafColumnTransformer extends ColumnTransformer { protected TsBlock input; @@ -39,4 +41,9 @@ public void checkType() { public void initFromTsBlock(TsBlock input) { this.input = input; } + + @Override + protected void forEachChild(Consumer action) { + // Leaf transformers have no children. + } } diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/MappableUDFColumnTransformer.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/MappableUDFColumnTransformer.java index 0d7576025c5e8..3712a74468b26 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/MappableUDFColumnTransformer.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/MappableUDFColumnTransformer.java @@ -26,7 +26,7 @@ import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.read.common.type.Type; -import java.util.Set; +import java.util.function.Consumer; public class MappableUDFColumnTransformer extends ColumnTransformer { @@ -96,17 +96,9 @@ public void close() { } @Override - public void clearCache() { - super.clearCache(); + protected void forEachChild(Consumer action) { for (ColumnTransformer columnTransformer : inputColumnTransformers) { - columnTransformer.clearCache(); - } - } - - @Override - protected void clearChildrenCacheForNewInput(Set visited) { - for (ColumnTransformer columnTransformer : inputColumnTransformers) { - columnTransformer.clearCacheForNewInput(visited); + action.accept(columnTransformer); } } } diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/MultiColumnTransformer.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/MultiColumnTransformer.java index 5d8be08896d91..afcd309069e87 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/MultiColumnTransformer.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/MultiColumnTransformer.java @@ -27,7 +27,7 @@ import java.util.ArrayList; import java.util.List; -import java.util.Set; +import java.util.function.Consumer; import java.util.stream.Collectors; public abstract class MultiColumnTransformer extends ColumnTransformer { @@ -91,18 +91,8 @@ public List getChildren() { } @Override - public void clearCache() { - super.clearCache(); - for (ColumnTransformer columnTransformer : columnTransformerList) { - columnTransformer.clearCache(); - } - } - - @Override - protected void clearChildrenCacheForNewInput(Set visited) { - for (ColumnTransformer columnTransformer : columnTransformerList) { - columnTransformer.clearCacheForNewInput(visited); - } + protected void forEachChild(Consumer action) { + columnTransformerList.forEach(action); } @Override diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/ternary/TernaryColumnTransformer.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/ternary/TernaryColumnTransformer.java index 1c8dfce914a8d..a92d59292a6e9 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/ternary/TernaryColumnTransformer.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/ternary/TernaryColumnTransformer.java @@ -25,7 +25,7 @@ import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.read.common.type.Type; -import java.util.Set; +import java.util.function.Consumer; public abstract class TernaryColumnTransformer extends ColumnTransformer { @@ -106,18 +106,10 @@ public ColumnTransformer getThirdColumnTransformer() { } @Override - public void clearCache() { - super.clearCache(); - firstColumnTransformer.clearCache(); - secondColumnTransformer.clearCache(); - thirdColumnTransformer.clearCache(); - } - - @Override - protected void clearChildrenCacheForNewInput(Set visited) { - firstColumnTransformer.clearCacheForNewInput(visited); - secondColumnTransformer.clearCacheForNewInput(visited); - thirdColumnTransformer.clearCacheForNewInput(visited); + protected void forEachChild(Consumer action) { + action.accept(firstColumnTransformer); + action.accept(secondColumnTransformer); + action.accept(thirdColumnTransformer); } @Override diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/unary/UnaryColumnTransformer.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/unary/UnaryColumnTransformer.java index 3e28d3cfe874a..1dcc6303534b6 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/unary/UnaryColumnTransformer.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/unary/UnaryColumnTransformer.java @@ -25,7 +25,7 @@ import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.read.common.type.Type; -import java.util.Set; +import java.util.function.Consumer; public abstract class UnaryColumnTransformer extends ColumnTransformer { protected ColumnTransformer childColumnTransformer; @@ -70,14 +70,8 @@ protected abstract void doTransform( Column column, ColumnBuilder columnBuilder, boolean[] selection); @Override - public void clearCache() { - super.clearCache(); - childColumnTransformer.clearCache(); - } - - @Override - protected void clearChildrenCacheForNewInput(Set visited) { - childColumnTransformer.clearCacheForNewInput(visited); + protected void forEachChild(Consumer action) { + action.accept(childColumnTransformer); } @Override diff --git a/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperatorTest.java b/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperatorTest.java index d50587fdea757..e21252dd87105 100644 --- a/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperatorTest.java +++ b/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperatorTest.java @@ -31,9 +31,12 @@ import org.apache.iotdb.calc.transformation.dag.column.leaf.IdentityColumnTransformer; import org.apache.iotdb.calc.transformation.dag.column.leaf.LeafColumnTransformer; import org.apache.iotdb.calc.transformation.dag.column.leaf.NullColumnTransformer; +import org.apache.iotdb.calc.transformation.dag.column.multi.MappableUDFColumnTransformer; +import org.apache.iotdb.calc.transformation.dag.udf.UDTFExecutor; import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNodeId; import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.read.common.block.TsBlock; import org.apache.tsfile.read.common.block.column.DoubleColumn; @@ -41,13 +44,14 @@ import org.apache.tsfile.read.common.block.column.TimeColumn; import org.junit.Test; +import java.time.ZoneId; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.Iterator; import java.util.List; import java.util.Optional; -import java.util.Set; +import java.util.function.Consumer; import static org.apache.tsfile.read.common.type.BooleanType.BOOLEAN; import static org.apache.tsfile.read.common.type.DoubleType.DOUBLE; @@ -265,6 +269,37 @@ public void testFilterAndCommonExpressionsShareCacheClearingTraversal() throws E } } + @Test + public void testMappableUdfClearsCaseDescendantsAcrossBlocks() throws Exception { + Graph graph = new Graph(); + ColumnTransformer input = graph.input(0); + ColumnTransformer caseWhen = + graph.caseWhen(graph.greaterThan(input, 0), graph.constant(1), graph.constant(0)); + ColumnTransformer udf = + new MappableUDFColumnTransformer( + DOUBLE, + new ColumnTransformer[] {reference(caseWhen), reference(input)}, + new UDTFExecutor("test", ZoneId.of("UTC")) { + @Override + public void execute(Column[] columns, ColumnBuilder builder) { + for (int i = 0; i < columns[0].getPositionCount(); i++) { + builder.writeDouble(columns[0].getDouble(i) + columns[1].getDouble(i)); + } + } + + @Override + public void beforeDestroy() { + // No UDF resources are allocated by this test executor. + } + }); + try (Operator operator = + project(graph, new TsBlock[] {block(1), block(-2, 3), block(4)}, udf)) { + assertBlock(operator.next(), new Double[] {2.0}); + assertBlock(operator.next(), new Double[] {-2.0, 4.0}); + assertBlock(operator.next(), new Double[] {5.0}); + } + } + @Test public void testReusingCasePreservesPendingSiblingColumn() throws Exception { Graph graph = new Graph(); @@ -415,7 +450,7 @@ protected void evaluate() { } @Override - protected void clearChildrenCacheForNewInput(Set visited) { + protected void forEachChild(Consumer action) { cacheClearCount++; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DeviceAttributeUpdater.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DeviceAttributeUpdater.java index 9120444ba0d23..98e0e66b50bfc 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DeviceAttributeUpdater.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DeviceAttributeUpdater.java @@ -40,8 +40,11 @@ import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; +import java.util.IdentityHashMap; import java.util.List; import java.util.Objects; +import java.util.Set; import java.util.function.BiFunction; import java.util.stream.Collectors; @@ -97,6 +100,8 @@ public void handleDeviceNode(final IDeviceMNode node) throws Metadata protected void update() throws MetadataException { final TsBlock block = getFilterTsBlock(); + Set visited = Collections.newSetFromMap(new IdentityHashMap<>()); + projectOutputTransformerList.forEach(transformer -> transformer.clearCacheForNewInput(visited)); projectLeafColumnTransformerList.forEach( leafColumnTransformer -> leafColumnTransformer.initFromTsBlock(block)); @@ -145,9 +150,13 @@ private TsBlock getFilterTsBlock() { final List filterResultColumns = new ArrayList<>(Arrays.asList(curBlock.getValueColumns())); - // Get result of calculated common sub expressions + // CASE evaluation can clear a common expression after the filter used it. Recompute it before + // materializing the column for projection within this same batch. commonTransformerList.forEach( - columnTransformer -> filterResultColumns.add(columnTransformer.getColumn())); + columnTransformer -> { + columnTransformer.tryEvaluate(); + filterResultColumns.add(columnTransformer.getColumn()); + }); final ColumnBuilder[] columnBuilders = filterTsBlockBuilder.getValueColumnBuilders(); final int rowCount = @@ -158,6 +167,12 @@ private TsBlock getFilterTsBlock() { return filterTsBlockBuilder.build(new TimeColumn(rowCount, new long[rowCount])); } + @Override + protected void clearFilterCacheForNewInput(Set visited) { + super.clearFilterCacheForNewInput(visited); + commonTransformerList.forEach(transformer -> transformer.clearCacheForNewInput(visited)); + } + @Override public void close() throws MetadataException { super.close(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DevicePredicateHandler.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DevicePredicateHandler.java index db7d0a2b374bf..600e6a2a9aaaf 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DevicePredicateHandler.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DevicePredicateHandler.java @@ -35,8 +35,11 @@ import org.apache.tsfile.read.common.block.TsBlockBuilder; import java.util.ArrayList; +import java.util.Collections; +import java.util.IdentityHashMap; import java.util.List; import java.util.Objects; +import java.util.Set; import java.util.stream.Collectors; import static org.apache.iotdb.calc.execution.operator.process.FilterAndProjectOperator.satisfy; @@ -106,6 +109,8 @@ public void prepareBatchResult() { return; } + // A CASE can leave shared expressions cached after the previous batch was consumed. + clearFilterCacheForNewInput(Collections.newSetFromMap(new IdentityHashMap<>())); // feed Filter ColumnTransformer, including TimeStampColumnTransformer and constant filterLeafColumnTransformerList.forEach( leafColumnTransformer -> leafColumnTransformer.initFromTsBlock(curBlock)); @@ -120,6 +125,10 @@ public void prepareBatchResult() { curFilterColumn = filterColumn; } + protected void clearFilterCacheForNewInput(Set visited) { + filterOutputTransformer.clearCacheForNewInput(visited); + } + protected boolean withoutFilter() { return Objects.isNull(filterOutputTransformer); } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DevicePredicateHandlerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DevicePredicateHandlerTest.java new file mode 100644 index 0000000000000..08b56188b2125 --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DevicePredicateHandlerTest.java @@ -0,0 +1,323 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.execution.operator.schema.source; + +import org.apache.iotdb.calc.transformation.dag.column.ColumnTransformer; +import org.apache.iotdb.calc.transformation.dag.column.TableCaseWhenThenColumnTransformer; +import org.apache.iotdb.calc.transformation.dag.column.binary.CompareEqualToColumnTransformer; +import org.apache.iotdb.calc.transformation.dag.column.leaf.ConstantColumnTransformer; +import org.apache.iotdb.calc.transformation.dag.column.leaf.IdentityColumnTransformer; +import org.apache.iotdb.calc.transformation.dag.column.leaf.LeafColumnTransformer; +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.schema.node.role.IDeviceMNode; +import org.apache.iotdb.commons.schema.table.TsTable; +import org.apache.iotdb.commons.schema.table.column.TagColumnSchema; +import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema; +import org.apache.iotdb.db.schemaengine.rescon.MemSchemaRegionStatistics; +import org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.mem.mnode.IMemMNode; +import org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.mem.mnode.info.TableDeviceInfo; +import org.apache.iotdb.db.schemaengine.schemaregion.read.resp.info.impl.ShowDevicesResult; + +import org.apache.tsfile.common.conf.TSFileConfig; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.block.column.BinaryColumn; +import org.apache.tsfile.read.common.block.column.BooleanColumn; +import org.apache.tsfile.utils.Binary; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Optional; + +import static org.apache.tsfile.read.common.type.BooleanType.BOOLEAN; +import static org.apache.tsfile.read.common.type.StringType.STRING; +import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class DevicePredicateHandlerTest { + + private static final List COLUMNS = + Collections.singletonList(new TagColumnSchema("tag", TSDataType.STRING)); + + @Test + public void testPredicateFilterDoesNotReuseFullBatches() throws Exception { + Graph graph = new Graph(); + ColumnTransformer predicate = graph.matchesCase(graph.input(0)); + try (DevicePredicateFilter filter = predicateFilter(graph, predicate)) { + int batchSize = DevicePredicateHandler.DEFAULT_MAX_TS_BLOCK_LINE_NUMBER; + for (String value : new String[] {"match", "other"}) { + for (int i = 0; i < batchSize; i++) { + filter.addBatch(device(value)); + } + // addBatch evaluates full batches automatically; exhausting the iterator resets the batch. + List result = consume(filter); + assertEquals(batchSize, result.size()); + for (String actual : result) { + assertEquals(value, actual); + } + } + } + } + + @Test + public void testPredicateFilterDoesNotReuseDifferentSizedBatches() throws Exception { + Graph graph = new Graph(); + ColumnTransformer predicate = graph.matchesCase(graph.input(0)); + try (DevicePredicateFilter filter = predicateFilter(graph, predicate)) { + assertEquals(Collections.singletonList("match"), filterBatch(filter, "match")); + assertEquals(Arrays.asList("other", "match"), filterBatch(filter, "other", "match")); + assertEquals(Collections.emptyList(), filterBatch(filter, "skip")); + } + } + + @Test + public void testBlackListConstructorDoesNotReusePredicateAcrossBatches() throws Exception { + Graph graph = new Graph(); + ColumnTransformer predicate = graph.matchesCase(graph.input(0)); + List> nodes = + Arrays.asList(node("match", 0), node("other", 1), node("match", 2)); + try (DeviceBlackListConstructor constructor = + new DeviceBlackListConstructor( + graph.leaves, + reference(predicate), + "table", + COLUMNS, + (pointer, name) -> null, + mock(MemSchemaRegionStatistics.class), + "test", + new TsTable("table"))) { + constructor.handleDeviceNode(nodes.get(0)); + constructor.prepareBatchResult(); + constructor.update(); + constructor.handleDeviceNode(nodes.get(1)); + constructor.handleDeviceNode(nodes.get(2)); + constructor.prepareBatchResult(); + constructor.update(); + assertEquals(3, constructor.getPreDeletedNum()); + for (IDeviceMNode node : nodes) { + verify(node).preDeactivateSelfOrTemplate(); + } + } + } + + @Test + public void testAttributeUpdaterDoesNotReuseFilterAcrossBatches() throws Exception { + Graph filter = new Graph(); + ColumnTransformer predicate = filter.matchesCase(filter.input(0)); + Graph projection = new Graph(); + List> updates = new ArrayList<>(); + try (DeviceAttributeUpdater updater = + attributeUpdater( + filter, predicate, Collections.emptyList(), projection, updates, projection.input(0))) { + updateBatch(updater, "match"); + updateBatch(updater, "other", "match"); + assertEquals( + Arrays.asList( + Arrays.asList("match", "match"), + Arrays.asList("other", "other"), + Arrays.asList("match", "match")), + updates); + } + } + + @Test + public void testAttributeUpdaterDoesNotReuseProjectionAcrossBatches() throws Exception { + Graph projection = new Graph(); + ColumnTransformer input = projection.input(0); + ColumnTransformer caseWhen = projection.caseResult(input); + List> updates = new ArrayList<>(); + try (DeviceAttributeUpdater updater = + attributeUpdater( + null, null, Collections.emptyList(), projection, updates, caseWhen, input)) { + updateBatch(updater, "match"); + updateBatch(updater, "other", "match"); + assertEquals( + Arrays.asList( + Arrays.asList("match", "match", "match"), + Arrays.asList("other", "other", "other"), + Arrays.asList("match", "match", "match")), + updates); + } + } + + @Test + public void testAttributeUpdaterRecomputesCommonExpressionClearedByCase() throws Exception { + Graph filter = new Graph(); + ColumnTransformer shared = filter.input(0); + ColumnTransformer predicate = + filter.caseWhen( + filter.equal(shared, filter.constant("match")), + filter.constant(true), + filter.constant(false)); + Graph projection = new Graph(); + List> updates = new ArrayList<>(); + try (DeviceAttributeUpdater updater = + attributeUpdater( + filter, + predicate, + Collections.singletonList(shared), + projection, + updates, + projection.input(1))) { + updateBatch(updater, "match"); + updateBatch(updater, "other", "match"); + assertEquals( + Arrays.asList(Arrays.asList("match", "match"), Arrays.asList("match", "match")), updates); + } + } + + private static DevicePredicateFilter predicateFilter(Graph graph, ColumnTransformer predicate) { + return new DevicePredicateFilter( + graph.leaves, reference(predicate), COLUMNS, "test", new TsTable("table")); + } + + private static List filterBatch(DevicePredicateFilter filter, String... values) { + for (String value : values) { + filter.addBatch(device(value)); + } + filter.prepareBatchResult(); + return consume(filter); + } + + private static List consume(DevicePredicateFilter filter) { + List result = new ArrayList<>(); + while (filter.hasNext()) { + result.add(filter.next().getRawNodes()[3]); + } + return result; + } + + private static DeviceAttributeUpdater attributeUpdater( + Graph filter, + ColumnTransformer predicate, + List common, + Graph projection, + List> updates, + ColumnTransformer... outputs) { + List projectOutputs = new ArrayList<>(); + for (ColumnTransformer output : outputs) { + projectOutputs.add(reference(output)); + } + common.forEach(ColumnTransformer::addReferenceCount); + return new DeviceAttributeUpdater( + Collections.nCopies(1 + common.size(), TSDataType.STRING), + filter == null ? Collections.emptyList() : filter.leaves, + predicate == null ? null : reference(predicate), + common, + COLUMNS, + projection.leaves, + projectOutputs, + (pointer, name) -> null, + (deviceId, pointer, values) -> { + List row = new ArrayList<>(); + row.add(deviceId[0]); + for (Object value : values) { + row.add(((Binary) value).getStringValue(TSFileConfig.STRING_CHARSET)); + } + updates.add(row); + }, + Collections.nCopies(outputs.length, "attribute"), + "test", + new TsTable("table")); + } + + private static void updateBatch(DeviceAttributeUpdater updater, String... values) + throws Exception { + for (int i = 0; i < values.length; i++) { + updater.handleDeviceNode(node(values[i], i)); + } + updater.prepareBatchResult(); + updater.update(); + } + + private static ShowDevicesResult device(String value) { + return new ShowDevicesResult(null, true, 0, new String[] {"root", "test", "table", value}); + } + + @SuppressWarnings("unchecked") + private static IDeviceMNode node(String value, int pointer) { + IDeviceMNode node = mock(IDeviceMNode.class); + TableDeviceInfo info = new TableDeviceInfo<>(); + info.setAttributePointer(pointer); + when(node.getDeviceInfo()).thenReturn(info); + when(node.getPartialPath()).thenReturn(new PartialPath(device(value).getRawNodes())); + when(node.isAlignedNullable()).thenReturn(true); + return node; + } + + private static ColumnTransformer reference(ColumnTransformer transformer) { + transformer.addReferenceCount(); + return transformer; + } + + private static class Graph { + private final List leaves = new ArrayList<>(); + + private ColumnTransformer input(int index) { + return leaf(new IdentityColumnTransformer(STRING, index)); + } + + private ColumnTransformer constant(String value) { + return leaf( + new ConstantColumnTransformer( + STRING, + new BinaryColumn( + 1, + Optional.empty(), + new Binary[] {new Binary(value, TSFileConfig.STRING_CHARSET)}))); + } + + private ColumnTransformer constant(boolean value) { + return leaf( + new ConstantColumnTransformer( + BOOLEAN, new BooleanColumn(1, Optional.empty(), new boolean[] {value}))); + } + + private ColumnTransformer leaf(LeafColumnTransformer transformer) { + leaves.add(transformer); + return transformer; + } + + private ColumnTransformer equal(ColumnTransformer left, ColumnTransformer right) { + return new CompareEqualToColumnTransformer(BOOLEAN, reference(left), reference(right)); + } + + private ColumnTransformer caseWhen( + ColumnTransformer when, ColumnTransformer then, ColumnTransformer otherwise) { + return new TableCaseWhenThenColumnTransformer( + then.getType(), + Collections.singletonList(reference(when)), + Collections.singletonList(reference(then)), + reference(otherwise)); + } + + private ColumnTransformer caseResult(ColumnTransformer input) { + return caseWhen(equal(input, constant("match")), constant("match"), constant("other")); + } + + private ColumnTransformer matchesCase(ColumnTransformer input) { + return equal(caseResult(input), input); + } + } +} From 50c38c16ddf29032a6d1cf1c89b2bcf67627e957 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Tue, 13 Oct 2026 11:27:51 +0800 Subject: [PATCH 5/8] Preserve Mockito compatibility for legacy cache clearing --- .../calc/transformation/dag/column/ColumnTransformer.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/ColumnTransformer.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/ColumnTransformer.java index e8dcbee6fb0fb..5189693d30e18 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/ColumnTransformer.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/ColumnTransformer.java @@ -115,7 +115,13 @@ public void close() { // do nothing } - public final void clearCache() { + /** + * Clears caches during evaluation. Subclasses should define child edges via {@link + * #forEachChild(Consumer)} instead of overriding this method. + * + *

This legacy entry point remains non-final for compatibility with existing test doubles. + */ + public void clearCache() { this.columnCache.clear(); if (shouldClearChildrenCache()) { forEachChild(ColumnTransformer::clearCache); From 5e0cb1c8a98345af99bb1115eebad1be7ea12ffc Mon Sep 17 00:00:00 2001 From: ColinLee Date: Tue, 13 Oct 2026 15:57:15 +0800 Subject: [PATCH 6/8] Precompute transformer cache reset nodes for each input phase --- .../process/FilterAndProjectOperator.java | 32 ++++--- .../dag/column/ColumnTransformer.java | 40 ++++++-- .../process/FilterAndProjectOperatorTest.java | 34 +++++-- .../dag/column/ColumnTransformerTest.java | 91 +++++++++++++++++++ .../schema/source/DeviceAttributeUpdater.java | 18 ++-- .../source/DeviceBlackListConstructor.java | 2 + .../schema/source/DevicePredicateFilter.java | 2 + .../schema/source/DevicePredicateHandler.java | 18 ++-- .../operator/schema/source/DeviceUpdater.java | 2 + 9 files changed, 191 insertions(+), 48 deletions(-) create mode 100644 iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/transformation/dag/column/ColumnTransformerTest.java diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperator.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperator.java index 7598a0c2196b8..03524b7ce6344 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperator.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperator.java @@ -47,11 +47,8 @@ import org.apache.tsfile.utils.RamUsageEstimator; import java.util.ArrayList; -import java.util.Collections; -import java.util.IdentityHashMap; import java.util.List; import java.util.OptionalInt; -import java.util.Set; public class FilterAndProjectOperator implements ProcessOperator { @@ -70,6 +67,10 @@ public class FilterAndProjectOperator implements ProcessOperator { private final List projectOutputTransformerList; + private final ColumnTransformer[] filterCacheTransformers; + + private final ColumnTransformer[] projectCacheTransformers; + private final TsBlockBuilder filterTsBlockBuilder; private final boolean hasNonMappableUDF; @@ -103,6 +104,13 @@ public FilterAndProjectOperator( this.hasNonMappableUDF = hasNonMappableUDF; this.filterTsBlockBuilder = new TsBlockBuilder(8, filterOutputDataTypes); this.hasFilter = hasFilter; + List filterRoots = new ArrayList<>(commonTransformerList); + if (filterOutputTransformer != null) { + filterRoots.add(filterOutputTransformer); + } + this.filterCacheTransformers = ColumnTransformer.collectTransformers(filterRoots); + this.projectCacheTransformers = + ColumnTransformer.collectTransformers(projectOutputTransformerList); } public FilterAndProjectOperator( @@ -114,6 +122,8 @@ public FilterAndProjectOperator( this.projectLeafColumnTransformerList = filterAndProjectOperator.projectLeafColumnTransformerList; this.projectOutputTransformerList = filterAndProjectOperator.projectOutputTransformerList; + this.filterCacheTransformers = filterAndProjectOperator.filterCacheTransformers; + this.projectCacheTransformers = filterAndProjectOperator.projectCacheTransformers; this.hasNonMappableUDF = filterAndProjectOperator.hasNonMappableUDF; this.hasFilter = filterAndProjectOperator.hasFilter; this.filterTsBlockBuilder = filterAndProjectOperator.filterTsBlockBuilder; @@ -158,10 +168,8 @@ private TsBlock getFilterTsBlock(TsBlock input) { // Short-circuit evaluation can reset cache reference counts when a shared expression is // recomputed. Invalidate the whole DAG before feeding a new block, including intermediate // expressions whose caches may still have unconsumed references from the previous block. - Set visited = Collections.newSetFromMap(new IdentityHashMap<>()); - filterOutputTransformer.clearCacheForNewInput(visited); - for (ColumnTransformer columnTransformer : commonTransformerList) { - columnTransformer.clearCacheForNewInput(visited); + for (ColumnTransformer columnTransformer : filterCacheTransformers) { + columnTransformer.clearCacheForNewInput(); } // feed Filter ColumnTransformer, including TimeStampColumnTransformer and constant for (LeafColumnTransformer leafColumnTransformer : filterLeafColumnTransformerList) { @@ -274,10 +282,10 @@ public static boolean satisfy(Column filterColumn, int rowIndex) { private TsBlock getTransformedTsBlock(TsBlock input) { final Column originTimeColumn = input.getTimeColumn(); final int positionCount = originTimeColumn.getPositionCount(); - // Clear all output DAGs before evaluating any output, visiting shared subexpressions only once. - Set visited = Collections.newSetFromMap(new IdentityHashMap<>()); - for (ColumnTransformer columnTransformer : projectOutputTransformerList) { - columnTransformer.clearCacheForNewInput(visited); + // The flat array was deduplicated at construction; clear each node before evaluating any + // output. + for (ColumnTransformer columnTransformer : projectCacheTransformers) { + columnTransformer.clearCacheForNewInput(); } // feed pre calculated data for (LeafColumnTransformer leafColumnTransformer : projectLeafColumnTransformerList) { @@ -455,6 +463,8 @@ public long ramBytesUsed() { return INSTANCE_SIZE + MemoryEstimationHelper.getEstimatedSizeOfAccountableObject(inputOperator) + MemoryEstimationHelper.getEstimatedSizeOfAccountableObject(operatorContext) + + RamUsageEstimator.shallowSizeOf(filterCacheTransformers) + + RamUsageEstimator.shallowSizeOf(projectCacheTransformers) + filterTsBlockBuilder.getRetainedSizeInBytes(); } } diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/ColumnTransformer.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/ColumnTransformer.java index 5189693d30e18..cb1d121c55c4f 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/ColumnTransformer.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/ColumnTransformer.java @@ -25,6 +25,10 @@ import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.read.common.type.TypeEnum; +import java.util.ArrayList; +import java.util.Collections; +import java.util.IdentityHashMap; +import java.util.List; import java.util.Set; import java.util.function.Consumer; @@ -129,18 +133,34 @@ public void clearCache() { } /** - * Clears cached columns throughout this DAG before initializing a new input block. Unlike {@link - * #clearCache()}, this also visits CASE descendants. Only call this before evaluating any - * expressions for the new block, when no sibling expression can still need a cached column. All - * roots in the same evaluation phase must share an identity-based visited set so shared - * subexpressions are cleared only once. + * Clears only this node's cached column before initializing a new input block. Call this on every + * node collected by {@link #collectTransformers(Iterable)} before evaluating any expressions for + * the new block, when no sibling expression can still need a cached column. */ - public final void clearCacheForNewInput(Set visited) { - if (!visited.add(this)) { - return; - } + public final void clearCacheForNewInput() { this.columnCache.clear(); - forEachChild(child -> child.clearCacheForNewInput(visited)); + } + + /** + * Collects all nodes reachable from the evaluation phase's roots, including CASE descendants, + * deduplicating by identity. Collect once after constructing the DAG and retain the flat array so + * new-input cache clearing needs no graph traversal or visited-set allocation. + */ + public static ColumnTransformer[] collectTransformers(Iterable roots) { + List transformers = new ArrayList<>(); + Set visited = Collections.newSetFromMap(new IdentityHashMap<>()); + for (ColumnTransformer root : roots) { + root.collectTransformers(transformers, visited); + } + return transformers.toArray(new ColumnTransformer[0]); + } + + private void collectTransformers( + List transformers, Set visited) { + if (visited.add(this)) { + transformers.add(this); + forEachChild(child -> child.collectTransformers(transformers, visited)); + } } protected boolean shouldClearChildrenCache() { diff --git a/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperatorTest.java b/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperatorTest.java index e21252dd87105..4b35a94c912b5 100644 --- a/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperatorTest.java +++ b/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperatorTest.java @@ -226,7 +226,23 @@ public void testProjectionReusesSharedExpressionsWithinEachBlock() throws Except } @Test - public void testProjectionClearsSharedDagOnlyOncePerBlock() throws Exception { + public void testCopiedOperatorReusesCollectedTransformers() throws Exception { + Graph graph = new Graph(); + CountingIdentityColumnTransformer input = new CountingIdentityColumnTransformer(); + graph.leaves.add(input); + try (FilterAndProjectOperator original = project(graph, new TsBlock[0], graph.add(input, 1)); + FilterAndProjectOperator copy = + new FilterAndProjectOperator( + original, new TsBlockSourceOperator(new TsBlock[] {block(1), block(2)}))) { + assertEquals(1, input.childTraversalCount); + assertBlock(copy.next(), new Double[] {2.0}); + assertBlock(copy.next(), new Double[] {3.0}); + assertEquals(1, input.childTraversalCount); + } + } + + @Test + public void testProjectionCollectsSharedDagOnlyOnce() throws Exception { Graph graph = new Graph(); CountingIdentityColumnTransformer input = new CountingIdentityColumnTransformer(); graph.leaves.add(input); @@ -238,15 +254,16 @@ public void testProjectionClearsSharedDagOnlyOncePerBlock() throws Exception { } try (Operator operator = project(graph, new TsBlock[] {block(1), block(2)}, shared, graph.add(shared, 1))) { + assertEquals(1, input.childTraversalCount); assertBlock(operator.next(), new Double[] {4096.0}, new Double[] {4097.0}); - assertEquals(1, input.cacheClearCount); + assertEquals(1, input.childTraversalCount); assertBlock(operator.next(), new Double[] {8192.0}, new Double[] {8193.0}); - assertEquals(2, input.cacheClearCount); + assertEquals(1, input.childTraversalCount); } } @Test - public void testFilterAndCommonExpressionsShareCacheClearingTraversal() throws Exception { + public void testFilterAndCommonExpressionsCollectSharedDagOnlyOnce() throws Exception { Graph filter = new Graph(); CountingIdentityColumnTransformer input = new CountingIdentityColumnTransformer(); filter.leaves.add(input); @@ -262,10 +279,11 @@ public void testFilterAndCommonExpressionsShareCacheClearingTraversal() throws E projection, projection.input(0), projection.input(1))) { + assertEquals(1, input.childTraversalCount); assertBlock(operator.next(), new Double[] {1.0}, new Double[] {2.0}); - assertEquals(1, input.cacheClearCount); + assertEquals(1, input.childTraversalCount); assertBlock(operator.next(), new Double[] {3.0}, new Double[] {4.0}); - assertEquals(2, input.cacheClearCount); + assertEquals(1, input.childTraversalCount); } } @@ -437,7 +455,7 @@ private ColumnTransformer caseWhen( private static class CountingIdentityColumnTransformer extends IdentityColumnTransformer { private int evaluationCount; - private int cacheClearCount; + private int childTraversalCount; private CountingIdentityColumnTransformer() { super(DOUBLE, 0); @@ -451,7 +469,7 @@ protected void evaluate() { @Override protected void forEachChild(Consumer action) { - cacheClearCount++; + childTraversalCount++; } } diff --git a/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/transformation/dag/column/ColumnTransformerTest.java b/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/transformation/dag/column/ColumnTransformerTest.java new file mode 100644 index 0000000000000..8c81d6f5ad8b0 --- /dev/null +++ b/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/transformation/dag/column/ColumnTransformerTest.java @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.calc.transformation.dag.column; + +import org.apache.iotdb.calc.transformation.dag.column.binary.ArithmeticAdditionColumnTransformer; +import org.apache.iotdb.calc.transformation.dag.column.leaf.IdentityColumnTransformer; + +import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.read.common.block.column.DoubleColumn; +import org.junit.Test; + +import java.util.Arrays; +import java.util.Optional; + +import static org.apache.tsfile.read.common.type.DoubleType.DOUBLE; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertSame; + +public class ColumnTransformerTest { + + @Test + public void testCollectionDeduplicatesByIdentity() { + ColumnTransformer first = new EqualIdentityColumnTransformer(0); + ColumnTransformer second = new EqualIdentityColumnTransformer(1); + ColumnTransformer root = new ArithmeticAdditionColumnTransformer(DOUBLE, first, second); + + ColumnTransformer[] collected = + ColumnTransformer.collectTransformers(Arrays.asList(root, first, root)); + + assertEquals(3, collected.length); + assertSame(root, collected[0]); + assertSame(first, collected[1]); + assertSame(second, collected[2]); + } + + @Test + public void testNewInputClearingOnlyClearsOwnCache() { + ColumnTransformer left = new IdentityColumnTransformer(DOUBLE, 0); + ColumnTransformer right = new IdentityColumnTransformer(DOUBLE, 1); + ColumnTransformer root = new ArithmeticAdditionColumnTransformer(DOUBLE, left, right); + left.addReferenceCount(); + right.addReferenceCount(); + root.addReferenceCount(); + left.initializeColumnCache(column(1)); + right.initializeColumnCache(column(2)); + root.initializeColumnCache(column(3)); + + root.clearCacheForNewInput(); + + assertEquals(0, root.getColumnCachePositionCount()); + assertEquals(1.0, left.getColumn().getDouble(0), 0.0); + assertEquals(2.0, right.getColumn().getDouble(0), 0.0); + } + + private static Column column(double value) { + return new DoubleColumn(1, Optional.empty(), new double[] {value}); + } + + private static class EqualIdentityColumnTransformer extends IdentityColumnTransformer { + private EqualIdentityColumnTransformer(int index) { + super(DOUBLE, index); + } + + @Override + public boolean equals(Object other) { + return other instanceof EqualIdentityColumnTransformer; + } + + @Override + public int hashCode() { + return 1; + } + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DeviceAttributeUpdater.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DeviceAttributeUpdater.java index 98e0e66b50bfc..04e08c9d133b7 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DeviceAttributeUpdater.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DeviceAttributeUpdater.java @@ -40,11 +40,8 @@ import java.util.ArrayList; import java.util.Arrays; -import java.util.Collections; -import java.util.IdentityHashMap; import java.util.List; import java.util.Objects; -import java.util.Set; import java.util.function.BiFunction; import java.util.stream.Collectors; @@ -54,6 +51,7 @@ public class DeviceAttributeUpdater extends DeviceUpdater { private final List commonTransformerList; private final List projectLeafColumnTransformerList; private final List projectOutputTransformerList; + private final ColumnTransformer[] projectCacheTransformers; private final TriConsumer attributeUpdater; private final TsBlockBuilder filterTsBlockBuilder; private final List attributePointers = new ArrayList<>(); @@ -78,6 +76,7 @@ public DeviceAttributeUpdater( super( filterLeafColumnTransformerList, filterOutputTransformer, + commonTransformerList, columnSchemaList, attributeProvider, database, @@ -85,6 +84,8 @@ public DeviceAttributeUpdater( this.commonTransformerList = commonTransformerList; this.projectLeafColumnTransformerList = projectLeafColumnTransformerList; this.projectOutputTransformerList = projectOutputTransformerList; + this.projectCacheTransformers = + ColumnTransformer.collectTransformers(projectOutputTransformerList); this.attributeUpdater = attributeUpdater; this.filterTsBlockBuilder = new TsBlockBuilder(filterOutputDataTypes); this.attributeNames = attributeNames; @@ -100,8 +101,9 @@ public void handleDeviceNode(final IDeviceMNode node) throws Metadata protected void update() throws MetadataException { final TsBlock block = getFilterTsBlock(); - Set visited = Collections.newSetFromMap(new IdentityHashMap<>()); - projectOutputTransformerList.forEach(transformer -> transformer.clearCacheForNewInput(visited)); + for (ColumnTransformer transformer : projectCacheTransformers) { + transformer.clearCacheForNewInput(); + } projectLeafColumnTransformerList.forEach( leafColumnTransformer -> leafColumnTransformer.initFromTsBlock(block)); @@ -167,12 +169,6 @@ private TsBlock getFilterTsBlock() { return filterTsBlockBuilder.build(new TimeColumn(rowCount, new long[rowCount])); } - @Override - protected void clearFilterCacheForNewInput(Set visited) { - super.clearFilterCacheForNewInput(visited); - commonTransformerList.forEach(transformer -> transformer.clearCacheForNewInput(visited)); - } - @Override public void close() throws MetadataException { super.close(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DeviceBlackListConstructor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DeviceBlackListConstructor.java index 3391f3e4080d8..4d3793dd44637 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DeviceBlackListConstructor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DeviceBlackListConstructor.java @@ -31,6 +31,7 @@ import org.apache.tsfile.utils.Binary; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.function.BiFunction; @@ -54,6 +55,7 @@ public DeviceBlackListConstructor( super( filterLeafColumnTransformerList, filterOutputTransformer, + Collections.emptyList(), columnSchemaList, attributeProvider, database, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DevicePredicateFilter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DevicePredicateFilter.java index 589f119c3ae14..a9db01b879a49 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DevicePredicateFilter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DevicePredicateFilter.java @@ -25,6 +25,7 @@ import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema; import org.apache.iotdb.db.schemaengine.schemaregion.read.resp.info.IDeviceSchemaInfo; +import java.util.Collections; import java.util.Iterator; import java.util.List; import java.util.NoSuchElementException; @@ -44,6 +45,7 @@ public DevicePredicateFilter( super( filterLeafColumnTransformerList, filterOutputTransformer, + Collections.emptyList(), columnSchemaList, database, table); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DevicePredicateHandler.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DevicePredicateHandler.java index 600e6a2a9aaaf..dd933d8403e75 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DevicePredicateHandler.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DevicePredicateHandler.java @@ -35,11 +35,8 @@ import org.apache.tsfile.read.common.block.TsBlockBuilder; import java.util.ArrayList; -import java.util.Collections; -import java.util.IdentityHashMap; import java.util.List; import java.util.Objects; -import java.util.Set; import java.util.stream.Collectors; import static org.apache.iotdb.calc.execution.operator.process.FilterAndProjectOperator.satisfy; @@ -48,6 +45,7 @@ public abstract class DevicePredicateHandler implements AutoCloseable { private final List filterLeafColumnTransformerList; protected final ColumnTransformer filterOutputTransformer; + private final ColumnTransformer[] filterCacheTransformers; private final List inputDataTypes; private final List columnSchemaList; private final int tagIndex; @@ -65,11 +63,17 @@ public abstract class DevicePredicateHandler implements AutoCloseable { protected DevicePredicateHandler( final List filterLeafColumnTransformerList, final ColumnTransformer filterOutputTransformer, + final List commonTransformerList, final List columnSchemaList, final String database, final TsTable table) { this.filterLeafColumnTransformerList = filterLeafColumnTransformerList; this.filterOutputTransformer = filterOutputTransformer; + List filterRoots = new ArrayList<>(commonTransformerList); + if (filterOutputTransformer != null) { + filterRoots.add(filterOutputTransformer); + } + this.filterCacheTransformers = ColumnTransformer.collectTransformers(filterRoots); this.columnSchemaList = columnSchemaList; this.inputDataTypes = columnSchemaList.stream() @@ -110,7 +114,9 @@ public void prepareBatchResult() { } // A CASE can leave shared expressions cached after the previous batch was consumed. - clearFilterCacheForNewInput(Collections.newSetFromMap(new IdentityHashMap<>())); + for (ColumnTransformer transformer : filterCacheTransformers) { + transformer.clearCacheForNewInput(); + } // feed Filter ColumnTransformer, including TimeStampColumnTransformer and constant filterLeafColumnTransformerList.forEach( leafColumnTransformer -> leafColumnTransformer.initFromTsBlock(curBlock)); @@ -125,10 +131,6 @@ public void prepareBatchResult() { curFilterColumn = filterColumn; } - protected void clearFilterCacheForNewInput(Set visited) { - filterOutputTransformer.clearCacheForNewInput(visited); - } - protected boolean withoutFilter() { return Objects.isNull(filterOutputTransformer); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DeviceUpdater.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DeviceUpdater.java index 2679f501dd060..5debad67608e3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DeviceUpdater.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DeviceUpdater.java @@ -41,6 +41,7 @@ public abstract class DeviceUpdater extends DevicePredicateHandler { protected DeviceUpdater( final List filterLeafColumnTransformerList, final ColumnTransformer filterOutputTransformer, + final List commonTransformerList, final List columnSchemaList, final BiFunction attributeProvider, final String database, @@ -48,6 +49,7 @@ protected DeviceUpdater( super( filterLeafColumnTransformerList, filterOutputTransformer, + commonTransformerList, columnSchemaList, database, table); From d7239384fd3a6200617de426fbf3d57930e6af87 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Wed, 14 Oct 2026 10:19:55 +0800 Subject: [PATCH 7/8] Respect filter selection when rebuilding common expression caches Share masked common-column materialization between query and schema UPDATE paths. Retain sibling columns before later evaluations can clear their caches, and read multi-argument sizes from retained columns. --- .../it/db/it/IoTDBCaseWhenThenTableIT.java | 66 +++++ .../it/schema/IoTDBCaseDeviceCacheIT.java | 79 ++++-- .../process/FilterAndProjectOperator.java | 31 +- .../dag/column/ColumnTransformer.java | 9 +- .../binary/BinaryColumnTransformer.java | 3 +- .../multi/MappableUDFColumnTransformer.java | 24 +- .../column/multi/MultiColumnTransformer.java | 17 +- .../ternary/TernaryColumnTransformer.java | 9 +- .../process/FilterAndProjectOperatorTest.java | 264 +++++++++++++----- .../schema/source/DeviceAttributeUpdater.java | 10 +- .../source/DevicePredicateHandlerTest.java | 7 +- 11 files changed, 382 insertions(+), 137 deletions(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBCaseWhenThenTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBCaseWhenThenTableIT.java index 169ebd3084ae3..0839c7e1f3415 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBCaseWhenThenTableIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBCaseWhenThenTableIT.java @@ -724,6 +724,72 @@ public void testCaseWhenReusesAggregationAcrossBlocks() { } } + @Test + public void testCaseOperandsAndMultiArgumentFunctions() { + prepareTableData( + new String[] { + "USE " + DATABASE, + "CREATE TABLE case_operands (dev STRING TAG, x DOUBLE FIELD)", + "INSERT INTO case_operands(time, dev, x) VALUES (0, 'd0', 1.0), (1, 'd0', -2.0)" + }); + String caseWhen = "CASE WHEN x > 0 THEN 2.0 ELSE 3.0 END"; + String[] expressions = { + "x + " + caseWhen, + "CASE WHEN x > -100 THEN coalesce(x, 0.0) END", + "greatest(x, " + caseWhen + ")", + "CASE WHEN x BETWEEN -5.0 AND " + caseWhen + " THEN 1.0 ELSE 0.0 END", + "CASE WHEN x > -100 THEN CASE WHEN x BETWEEN -5.0 AND " + + caseWhen + + " THEN 1.0 ELSE 0.0 END END", + "coalesce(x, " + caseWhen + ")", + "CASE WHEN x > -100 THEN coalesce(x, " + caseWhen + ") END" + }; + String[][] results = { + {"3.0,", "1.0,"}, + {"1.0,", "-2.0,"}, + {"2.0,", "3.0,"}, + {"1.0,", "1.0,"}, + {"1.0,", "1.0,"}, + {"1.0,", "-2.0,"}, + {"1.0,", "-2.0,"} + }; + for (int i = 0; i < expressions.length; i++) { + tableResultSetEqualTest( + "SELECT " + expressions[i] + " AS result FROM case_operands ORDER BY time", + new String[] {"result"}, + results[i], + DATABASE); + } + tableResultSetEqualTest( + "SELECT coalesce(x, 0.0) AS result FROM case_operands " + + "WHERE CASE WHEN coalesce(x, 0.0) > 0 THEN 1.0 ELSE 0.0 END > 0", + new String[] {"result"}, + new String[] {"1.0,"}, + DATABASE); + } + + @Test + public void testCommonCastRespectsCaseFilter() { + prepareTableData( + new String[] { + "USE " + DATABASE, + "CREATE TABLE guarded_common_cast (dev STRING TAG, src STRING FIELD)", + "INSERT INTO guarded_common_cast(time, dev, src) VALUES " + + "(0, 'd0', 'n/a'), (1, 'd0', '5')" + }); + String query = + "SELECT CAST(src AS INT32) + 1 AS result FROM guarded_common_cast " + + "WHERE CASE WHEN src <> 'n/a' THEN CAST(src AS INT32) > 10 ELSE false END"; + tableResultSetEqualTest(query, new String[] {"result"}, new String[0], DATABASE); + + prepareTableData( + new String[] { + "USE " + DATABASE, + "INSERT INTO guarded_common_cast(time, dev, src) VALUES (2, 'd0', '50')" + }); + tableResultSetEqualTest(query, new String[] {"result"}, new String[] {"51,"}, DATABASE); + } + @Test public void testReusingCasePreservesPendingSiblingColumn() { prepareTableData( diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBCaseDeviceCacheIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBCaseDeviceCacheIT.java index 3ef2464181106..6596f17f4274e 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBCaseDeviceCacheIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBCaseDeviceCacheIT.java @@ -66,20 +66,21 @@ public void testDevicePredicatesAcrossBatches() throws Exception { try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); Statement statement = connection.createStatement()) { createTable(statement, "predicate_cache"); - String predicate = CASE_RESULT + " = src"; - assertResultSetEqual( - statement.executeQuery("SHOW DEVICES FROM predicate_cache WHERE " + predicate), - "id,src,a,b,", - new HashSet<>( - Arrays.asList( - "d0,match,null,null,", - "d1,match,null,null,", - "d2,other,null,null,", - "d3,other,null,null,"))); - assertResultSetEqual( - statement.executeQuery("COUNT DEVICES FROM predicate_cache WHERE " + predicate), - "count(devices),", - Collections.singleton("4,")); + for (String predicate : new String[] {CASE_RESULT + " = src", "src = " + CASE_RESULT}) { + assertResultSetEqual( + statement.executeQuery("SHOW DEVICES FROM predicate_cache WHERE " + predicate), + "id,src,a,b,", + new HashSet<>( + Arrays.asList( + "d0,match,null,null,", + "d1,match,null,null,", + "d2,other,null,null,", + "d3,other,null,null,"))); + assertResultSetEqual( + statement.executeQuery("COUNT DEVICES FROM predicate_cache WHERE " + predicate), + "count(devices),", + Collections.singleton("4,")); + } } } @@ -98,6 +99,18 @@ public void testAttributeProjectionAcrossBatches() throws Exception { "d1,match,match,match,", "d2,other,other,other,", "d3,other,other,other,"))); + statement.execute( + "UPDATE projection_cache SET a = " + + "concat(src, CASE WHEN src = 'match' THEN '-m' ELSE '-o' END)"); + assertResultSetEqual( + statement.executeQuery("SHOW DEVICES FROM projection_cache"), + "id,src,a,b,", + new HashSet<>( + Arrays.asList( + "d0,match,match-m,match,", + "d1,match,match-m,match,", + "d2,other,other-o,other,", + "d3,other,other-o,other,"))); } } @@ -107,20 +120,50 @@ public void testAttributeCommonExpressionClearedByCase() throws Exception { Statement statement = connection.createStatement()) { createTable(statement, "common_cache"); statement.execute( - "UPDATE common_cache SET a = src " - + "WHERE CASE WHEN src = 'match' THEN true ELSE false END"); + "UPDATE common_cache SET a = upper(src) " + + "WHERE CASE WHEN upper(src) = 'MATCH' THEN true ELSE false END"); assertResultSetEqual( statement.executeQuery("SHOW DEVICES FROM common_cache"), "id,src,a,b,", new HashSet<>( Arrays.asList( - "d0,match,match,null,", - "d1,match,match,null,", + "d0,match,MATCH,null,", + "d1,match,MATCH,null,", "d2,other,null,null,", "d3,other,null,null,"))); } } + @Test + public void testGuardedCommonCastInAttributeUpdate() throws Exception { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("USE " + DATABASE); + statement.execute( + "CREATE TABLE guarded_cast " + + "(id STRING TAG, src STRING ATTRIBUTE, a STRING ATTRIBUTE, v DOUBLE FIELD)"); + statement.execute( + "INSERT INTO guarded_cast(time, id, src, a, v) VALUES " + + "(1, 'd0', 'n/a', 'seed', 1), (2, 'd1', '5', 'seed', 1)"); + String update = + "UPDATE guarded_cast SET a = CAST(CAST(src AS INT32) + 1 AS STRING) " + + "WHERE CASE WHEN src <> 'n/a' THEN CAST(src AS INT32) > 10 ELSE false END"; + + statement.execute(update); + assertResultSetEqual( + statement.executeQuery("SHOW DEVICES FROM guarded_cast"), + "id,src,a,", + new HashSet<>(Arrays.asList("d0,n/a,seed,", "d1,5,seed,"))); + + statement.execute("UPDATE guarded_cast SET src = '50' WHERE id = 'd1'"); + statement.execute(update); + assertResultSetEqual( + statement.executeQuery("SHOW DEVICES FROM guarded_cast"), + "id,src,a,", + new HashSet<>(Arrays.asList("d0,n/a,seed,", "d1,50,51,"))); + } + } + private static void createTable(Statement statement, String table) throws SQLException { statement.execute("USE " + DATABASE); statement.execute( diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperator.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperator.java index 03524b7ce6344..b258ee71ebf1f 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperator.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperator.java @@ -189,13 +189,8 @@ private TsBlock getFilterTsBlock(TsBlock input) { } if (!hasNonMappableUDF) { - // get result of calculated common sub expressions - for (ColumnTransformer columnTransformer : commonTransformerList) { - // CASE evaluation can clear a common expression after the filter used it. Recompute it - // before materializing the column for projection within this same input block. - columnTransformer.tryEvaluate(); - resultColumns.add(columnTransformer.getColumn()); - } + appendCommonExpressionColumns( + resultColumns, commonTransformerList, filterColumn, positionCount); } final ColumnBuilder[] columnBuilders = filterTsBlockBuilder.getValueColumnBuilders(); @@ -222,6 +217,28 @@ private TsBlock getFilterTsBlock(TsBlock input) { } } + /** + * Materializes common expressions for projection after filtering. CASE may have invalidated their + * caches, so any recomputation must preserve the filter mask and skip rejected input rows. + */ + public static void appendCommonExpressionColumns( + List resultColumns, + List commonTransformers, + Column filterColumn, + int positionCount) { + if (commonTransformers.isEmpty()) { + return; + } + boolean[] selection = new boolean[positionCount]; + for (int i = 0; i < positionCount; i++) { + selection[i] = satisfy(filterColumn, i); + } + for (ColumnTransformer transformer : commonTransformers) { + transformer.tryEvaluateWithSelection(selection); + resultColumns.add(transformer.getColumn()); + } + } + private static int constructFilteredTsBlock( List resultColumns, TimeColumnBuilder timeBuilder, diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/ColumnTransformer.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/ColumnTransformer.java index cb1d121c55c4f..d21b89b783db9 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/ColumnTransformer.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/ColumnTransformer.java @@ -54,6 +54,13 @@ public void tryEvaluate() { } } + /** Reuses the cached column, or evaluates with the selection if it was invalidated. */ + public void tryEvaluateWithSelection(boolean[] selection) { + if (!columnCache.hasCached()) { + evaluateWithSelection(selection); + } + } + public Column getColumn() { return columnCache.getColumn(); } @@ -123,7 +130,7 @@ public void close() { * Clears caches during evaluation. Subclasses should define child edges via {@link * #forEachChild(Consumer)} instead of overriding this method. * - *

This legacy entry point remains non-final for compatibility with existing test doubles. + *

Kept non-final so existing Mockito-based tests can stub it. */ public void clearCache() { this.columnCache.clear(); diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/binary/BinaryColumnTransformer.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/binary/BinaryColumnTransformer.java index d2ca5270dee0f..d1715b12c3c8a 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/binary/BinaryColumnTransformer.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/binary/BinaryColumnTransformer.java @@ -44,10 +44,11 @@ protected BinaryColumnTransformer( @Override public void evaluate() { leftTransformer.tryEvaluate(); - rightTransformer.tryEvaluate(); // attention: get positionCount before calling getColumn int positionCount = leftTransformer.getColumnCachePositionCount(); Column leftColumn = leftTransformer.getColumn(); + // A later child can clear shared caches; retain each column before evaluating its sibling. + rightTransformer.tryEvaluate(); Column rightColumn = rightTransformer.getColumn(); ColumnBuilder builder = returnType.createColumnBuilder(positionCount); diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/MappableUDFColumnTransformer.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/MappableUDFColumnTransformer.java index 3712a74468b26..9826c6a9b0428 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/MappableUDFColumnTransformer.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/MappableUDFColumnTransformer.java @@ -44,30 +44,30 @@ public MappableUDFColumnTransformer( @Override public void evaluate() { // pull columns from previous transformers - for (ColumnTransformer inputColumnTransformer : inputColumnTransformers) { - inputColumnTransformer.tryEvaluate(); + Column[] columns = new Column[inputColumnTransformers.length]; + for (int i = 0; i < inputColumnTransformers.length; i++) { + inputColumnTransformers[i].tryEvaluate(); + // A later child can clear shared caches; retain each column before evaluating its sibling. + columns[i] = inputColumnTransformers[i].getColumn(); } - doTransform(); + doTransform(columns); } @Override public void evaluateWithSelection(boolean[] selection) { - for (ColumnTransformer inputColumnTransformer : inputColumnTransformers) { - inputColumnTransformer.evaluateWithSelection(selection); + Column[] columns = new Column[inputColumnTransformers.length]; + for (int i = 0; i < inputColumnTransformers.length; i++) { + inputColumnTransformers[i].evaluateWithSelection(selection); + columns[i] = inputColumnTransformers[i].getColumn(); } - doTransform(); + doTransform(columns); // clear cache for (ColumnTransformer inputColumnTransformer : inputColumnTransformers) { inputColumnTransformer.clearCache(); } } - private void doTransform() { - int size = inputColumnTransformers.length; - Column[] columns = new Column[size]; - for (int i = 0; i < size; i++) { - columns[i] = inputColumnTransformers[i].getColumn(); - } + private void doTransform(Column[] columns) { // construct input TsBlock with columns int positionCount = columns[0].getPositionCount(); ColumnBuilder builder = returnType.createColumnBuilder(positionCount); diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/MultiColumnTransformer.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/MultiColumnTransformer.java index afcd309069e87..cfd9c8913f24b 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/MultiColumnTransformer.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/MultiColumnTransformer.java @@ -28,7 +28,6 @@ import java.util.ArrayList; import java.util.List; import java.util.function.Consumer; -import java.util.stream.Collectors; public abstract class MultiColumnTransformer extends ColumnTransformer { @@ -43,20 +42,17 @@ protected MultiColumnTransformer(Type returnType, List column @Override public void evaluate() { + List childrenColumns = new ArrayList<>(columnTransformerList.size()); for (ColumnTransformer child : columnTransformerList) { child.tryEvaluate(); + // A later child can clear shared caches; retain each column before evaluating its sibling. + childrenColumns.add(child.getColumn()); } - // attention: get positionCount before calling getColumn - int positionCount = columnTransformerList.get(0).getColumnCachePositionCount(); + int positionCount = childrenColumns.get(0).getPositionCount(); ColumnBuilder builder = returnType.createColumnBuilder(positionCount); - doTransform( - columnTransformerList.stream() - .map(ColumnTransformer::getColumn) - .collect(Collectors.toList()), - builder, - positionCount); + doTransform(childrenColumns, builder, positionCount); initializeColumnCache(builder.build()); } @@ -69,7 +65,8 @@ public void evaluateWithSelection(boolean[] selection) { childrenColumns.add(child.getColumn()); } - int positionCount = columnTransformerList.get(0).getColumnCachePositionCount(); + // getColumn() can release the first child's cache; read the retained column's size instead. + int positionCount = childrenColumns.get(0).getPositionCount(); ColumnBuilder builder = returnType.createColumnBuilder(positionCount); doTransform(childrenColumns, builder, positionCount, selection); diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/ternary/TernaryColumnTransformer.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/ternary/TernaryColumnTransformer.java index a92d59292a6e9..a1b7c91357cd3 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/ternary/TernaryColumnTransformer.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/ternary/TernaryColumnTransformer.java @@ -50,11 +50,12 @@ protected TernaryColumnTransformer( @Override public void evaluate() { firstColumnTransformer.tryEvaluate(); - secondColumnTransformer.tryEvaluate(); - thirdColumnTransformer.tryEvaluate(); int positionCount = firstColumnTransformer.getColumnCachePositionCount(); Column firstColumn = firstColumnTransformer.getColumn(); + // A later child can clear shared caches; retain each column before evaluating its sibling. + secondColumnTransformer.tryEvaluate(); Column secondColumn = secondColumnTransformer.getColumn(); + thirdColumnTransformer.tryEvaluate(); Column thirdColumn = thirdColumnTransformer.getColumn(); ColumnBuilder columnBuilder = returnType.createColumnBuilder(positionCount); doTransform(firstColumn, secondColumn, thirdColumn, columnBuilder, positionCount); @@ -64,11 +65,11 @@ public void evaluate() { @Override public void evaluateWithSelection(boolean[] selection) { firstColumnTransformer.evaluateWithSelection(selection); - secondColumnTransformer.evaluateWithSelection(selection); - thirdColumnTransformer.evaluateWithSelection(selection); int positionCount = firstColumnTransformer.getColumnCachePositionCount(); Column firstColumn = firstColumnTransformer.getColumn(); + secondColumnTransformer.evaluateWithSelection(selection); Column secondColumn = secondColumnTransformer.getColumn(); + thirdColumnTransformer.evaluateWithSelection(selection); Column thirdColumn = thirdColumnTransformer.getColumn(); ColumnBuilder columnBuilder = returnType.createColumnBuilder(positionCount); doTransform(firstColumn, secondColumn, thirdColumn, columnBuilder, positionCount, selection); diff --git a/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperatorTest.java b/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperatorTest.java index 4b35a94c912b5..0ba2c4d7ef50c 100644 --- a/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperatorTest.java +++ b/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperatorTest.java @@ -27,34 +27,43 @@ import org.apache.iotdb.calc.transformation.dag.column.TableCaseWhenThenColumnTransformer; import org.apache.iotdb.calc.transformation.dag.column.binary.ArithmeticAdditionColumnTransformer; import org.apache.iotdb.calc.transformation.dag.column.binary.CompareGreaterThanColumnTransformer; +import org.apache.iotdb.calc.transformation.dag.column.binary.CompareNonEqualColumnTransformer; import org.apache.iotdb.calc.transformation.dag.column.leaf.ConstantColumnTransformer; import org.apache.iotdb.calc.transformation.dag.column.leaf.IdentityColumnTransformer; import org.apache.iotdb.calc.transformation.dag.column.leaf.LeafColumnTransformer; import org.apache.iotdb.calc.transformation.dag.column.leaf.NullColumnTransformer; +import org.apache.iotdb.calc.transformation.dag.column.multi.CoalesceColumnTransformer; import org.apache.iotdb.calc.transformation.dag.column.multi.MappableUDFColumnTransformer; +import org.apache.iotdb.calc.transformation.dag.column.ternary.BetweenColumnTransformer; +import org.apache.iotdb.calc.transformation.dag.column.unary.scalar.CastFunctionColumnTransformer; import org.apache.iotdb.calc.transformation.dag.udf.UDTFExecutor; import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNodeId; import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; +import org.apache.tsfile.common.conf.TSFileConfig; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.read.common.block.TsBlock; +import org.apache.tsfile.read.common.block.column.BinaryColumn; +import org.apache.tsfile.read.common.block.column.BooleanColumn; import org.apache.tsfile.read.common.block.column.DoubleColumn; import org.apache.tsfile.read.common.block.column.RunLengthEncodedColumn; import org.apache.tsfile.read.common.block.column.TimeColumn; +import org.apache.tsfile.utils.Binary; import org.junit.Test; import java.time.ZoneId; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; -import java.util.Iterator; import java.util.List; import java.util.Optional; import java.util.function.Consumer; import static org.apache.tsfile.read.common.type.BooleanType.BOOLEAN; import static org.apache.tsfile.read.common.type.DoubleType.DOUBLE; +import static org.apache.tsfile.read.common.type.IntType.INT32; +import static org.apache.tsfile.read.common.type.StringType.STRING; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -166,6 +175,153 @@ public void testFilterCommonExpressionsDoNotCrossBlocks() throws Exception { } } + @Test + public void testCommonCastOnlyEvaluatesRowsPassingCaseFilter() throws Exception { + for (boolean runLengthEncodedTime : new boolean[] {false, true}) { + Graph filter = new Graph(); + ColumnTransformer input = filter.leaf(new IdentityColumnTransformer(STRING, 0)); + ColumnTransformer invalid = + filter.leaf( + new ConstantColumnTransformer( + STRING, + new BinaryColumn( + 1, + Optional.empty(), + new Binary[] {new Binary("n/a", TSFileConfig.STRING_CHARSET)}))); + ColumnTransformer guard = + new CompareNonEqualColumnTransformer(BOOLEAN, reference(input), reference(invalid)); + ColumnTransformer common = + new CastFunctionColumnTransformer(INT32, reference(input), ZoneId.of("UTC")); + ColumnTransformer otherwise = + filter.leaf( + new ConstantColumnTransformer( + BOOLEAN, new BooleanColumn(1, Optional.empty(), new boolean[] {false}))); + ColumnTransformer predicate = + new TableCaseWhenThenColumnTransformer( + BOOLEAN, + Collections.singletonList(reference(guard)), + Collections.singletonList(reference(filter.greaterThan(common, 10))), + reference(otherwise)); + IdentityColumnTransformer output = new IdentityColumnTransformer(INT32, 1); + reference(common); + try (Operator operator = + new FilterAndProjectOperator( + new TestOperatorContext(), + new ValuesOperator( + new TestOperatorContext(), + Arrays.asList( + stringBlock(runLengthEncodedTime, "n/a", "5"), + stringBlock(runLengthEncodedTime, "n/a", "50", "5"), + stringBlock(runLengthEncodedTime, "n/a", "60"))), + Arrays.asList(TSDataType.STRING, TSDataType.INT32), + filter.leaves, + reference(predicate), + Collections.singletonList(common), + Collections.singletonList(output), + Collections.singletonList(reference(output)), + false, + true)) { + assertEquals(0, operator.next().getPositionCount()); + TsBlock firstMatch = operator.next(); + assertEquals(1, firstMatch.getPositionCount()); + assertEquals(50, firstMatch.getColumn(0).getInt(0)); + TsBlock secondMatch = operator.next(); + assertEquals(1, secondMatch.getPositionCount()); + assertEquals(60, secondMatch.getColumn(0).getInt(0)); + } + } + } + + @Test + public void testCommonMultiArgumentExpressionRespectsCaseFilter() throws Exception { + Graph filter = new Graph(); + ColumnTransformer common = + new CoalesceColumnTransformer( + DOUBLE, Arrays.asList(reference(filter.input(0)), reference(filter.constant(0)))); + ColumnTransformer caseWhen = + filter.caseWhen(filter.greaterThan(common, 0), filter.constant(1), filter.constant(0)); + Graph projection = new Graph(); + try (Operator operator = + operator( + new TsBlock[] {block(-2, 3), block(4, -5)}, + filter, + filter.greaterThan(caseWhen, 0), + Collections.singletonList(common), + projection, + projection.input(1))) { + assertBlock(operator.next(), new Double[] {3.0}); + assertBlock(operator.next(), new Double[] {4.0}); + } + } + + @Test + public void testBinaryPreservesSiblingColumnBeforeCase() throws Exception { + Graph graph = new Graph(); + ColumnTransformer input = graph.input(0); + ColumnTransformer caseWhen = + graph.caseWhen(graph.greaterThan(input, 0), graph.constant(2), graph.constant(3)); + try (Operator operator = + project(graph, new TsBlock[] {block(1, -2)}, graph.add(input, caseWhen))) { + assertBlock(operator.next(), new Double[] {3.0, 1.0}); + } + } + + @Test + public void testMultiArgumentFunctionPreservesSiblingColumnBeforeCase() throws Exception { + for (boolean selected : new boolean[] {false, true}) { + Graph graph = new Graph(); + ColumnTransformer input = graph.input(0); + ColumnTransformer caseWhen = + graph.caseWhen(graph.greaterThan(input, 0), graph.constant(2), graph.constant(3)); + ColumnTransformer output = + new CoalesceColumnTransformer( + DOUBLE, Arrays.asList(reference(input), reference(caseWhen))); + if (selected) { + output = graph.caseWhen(graph.greaterThan(graph.constant(1), 0), output, graph.nullValue()); + } + try (Operator operator = project(graph, new TsBlock[] {block(1, -2)}, output)) { + assertBlock(operator.next(), new Double[] {1.0, -2.0}); + } + } + } + + @Test + public void testTernaryPreservesSiblingColumnsBeforeCase() throws Exception { + for (boolean selected : new boolean[] {false, true}) { + Graph graph = new Graph(); + ColumnTransformer input = graph.input(0); + ColumnTransformer caseWhen = + graph.caseWhen(graph.greaterThan(input, 0), graph.constant(2), graph.constant(3)); + ColumnTransformer between = + new BetweenColumnTransformer( + BOOLEAN, reference(input), reference(graph.constant(-5)), reference(caseWhen), false); + ColumnTransformer output = graph.caseWhen(between, graph.constant(1), graph.constant(0)); + if (selected) { + output = graph.caseWhen(graph.greaterThan(graph.constant(1), 0), output, graph.nullValue()); + } + try (Operator operator = project(graph, new TsBlock[] {block(1, -2)}, output)) { + assertBlock(operator.next(), new Double[] {1.0, 1.0}); + } + } + } + + @Test + public void testMappableUdfPreservesSiblingColumnsBeforeCase() throws Exception { + for (boolean selected : new boolean[] {false, true}) { + Graph graph = new Graph(); + ColumnTransformer input = graph.input(0); + ColumnTransformer caseWhen = + graph.caseWhen(graph.greaterThan(input, 0), graph.constant(2), graph.constant(3)); + ColumnTransformer output = sumUdf(input, caseWhen); + if (selected) { + output = graph.caseWhen(graph.greaterThan(graph.constant(1), 0), output, graph.nullValue()); + } + try (Operator operator = project(graph, new TsBlock[] {block(1, -2)}, output)) { + assertBlock(operator.next(), new Double[] {3.0, 1.0}); + } + } + } + @Test public void testProjectionClearsCachedDescendantsOfNestedCase() throws Exception { Graph graph = new Graph(); @@ -233,7 +389,8 @@ public void testCopiedOperatorReusesCollectedTransformers() throws Exception { try (FilterAndProjectOperator original = project(graph, new TsBlock[0], graph.add(input, 1)); FilterAndProjectOperator copy = new FilterAndProjectOperator( - original, new TsBlockSourceOperator(new TsBlock[] {block(1), block(2)}))) { + original, + new ValuesOperator(new TestOperatorContext(), Arrays.asList(block(1), block(2))))) { assertEquals(1, input.childTraversalCount); assertBlock(copy.next(), new Double[] {2.0}); assertBlock(copy.next(), new Double[] {3.0}); @@ -293,23 +450,7 @@ public void testMappableUdfClearsCaseDescendantsAcrossBlocks() throws Exception ColumnTransformer input = graph.input(0); ColumnTransformer caseWhen = graph.caseWhen(graph.greaterThan(input, 0), graph.constant(1), graph.constant(0)); - ColumnTransformer udf = - new MappableUDFColumnTransformer( - DOUBLE, - new ColumnTransformer[] {reference(caseWhen), reference(input)}, - new UDTFExecutor("test", ZoneId.of("UTC")) { - @Override - public void execute(Column[] columns, ColumnBuilder builder) { - for (int i = 0; i < columns[0].getPositionCount(); i++) { - builder.writeDouble(columns[0].getDouble(i) + columns[1].getDouble(i)); - } - } - - @Override - public void beforeDestroy() { - // No UDF resources are allocated by this test executor. - } - }); + ColumnTransformer udf = sumUdf(caseWhen, input); try (Operator operator = project(graph, new TsBlock[] {block(1), block(-2, 3), block(4)}, udf)) { assertBlock(operator.next(), new Double[] {2.0}); @@ -318,6 +459,25 @@ public void beforeDestroy() { } } + private static ColumnTransformer sumUdf(ColumnTransformer left, ColumnTransformer right) { + return new MappableUDFColumnTransformer( + DOUBLE, + new ColumnTransformer[] {reference(left), reference(right)}, + new UDTFExecutor("test", ZoneId.of("UTC")) { + @Override + public void execute(Column[] columns, ColumnBuilder builder) { + for (int i = 0; i < columns[0].getPositionCount(); i++) { + builder.writeDouble(columns[0].getDouble(i) + columns[1].getDouble(i)); + } + } + + @Override + public void beforeDestroy() { + // No UDF resources are allocated by this test executor. + } + }); + } + @Test public void testReusingCasePreservesPendingSiblingColumn() throws Exception { Graph graph = new Graph(); @@ -360,7 +520,7 @@ private static FilterAndProjectOperator operator( } return new FilterAndProjectOperator( new TestOperatorContext(), - new TsBlockSourceOperator(blocks), + new ValuesOperator(new TestOperatorContext(), Arrays.asList(blocks)), Collections.nCopies(1 + common.size(), TSDataType.DOUBLE), filter == null ? Collections.emptyList() : filter.leaves, predicate == null ? null : reference(predicate), @@ -384,6 +544,16 @@ private static TsBlock block(double... values) { return block(false, values); } + private static TsBlock stringBlock(boolean runLengthEncodedTime, String... values) { + Binary[] binaries = + Arrays.stream(values) + .map(value -> new Binary(value, TSFileConfig.STRING_CHARSET)) + .toArray(Binary[]::new); + return new TsBlock( + block(runLengthEncodedTime, new double[values.length]).getTimeColumn(), + new BinaryColumn(values.length, Optional.empty(), binaries)); + } + private static TsBlock block(boolean runLengthEncodedTime, double... values) { Column timeColumn = runLengthEncodedTime @@ -498,58 +668,4 @@ public long ramBytesUsed() { return 0; } } - - private static class TsBlockSourceOperator implements Operator { - private final Iterator blocks; - private final CommonOperatorContext context = new TestOperatorContext(); - - private TsBlockSourceOperator(TsBlock[] blocks) { - this.blocks = Arrays.asList(blocks).iterator(); - } - - @Override - public CommonOperatorContext getOperatorContext() { - return context; - } - - @Override - public TsBlock next() { - return blocks.next(); - } - - @Override - public boolean hasNext() { - return blocks.hasNext(); - } - - @Override - public boolean isFinished() { - return !hasNext(); - } - - @Override - public void close() { - // No resources to release. - } - - @Override - public long calculateMaxPeekMemory() { - return 0; - } - - @Override - public long calculateMaxReturnSize() { - return 0; - } - - @Override - public long calculateRetainedSizeAfterCallingNext() { - return 0; - } - - @Override - public long ramBytesUsed() { - return 0; - } - } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DeviceAttributeUpdater.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DeviceAttributeUpdater.java index 04e08c9d133b7..8d42d07b2a374 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DeviceAttributeUpdater.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DeviceAttributeUpdater.java @@ -45,6 +45,7 @@ import java.util.function.BiFunction; import java.util.stream.Collectors; +import static org.apache.iotdb.calc.execution.operator.process.FilterAndProjectOperator.appendCommonExpressionColumns; import static org.apache.iotdb.calc.execution.operator.process.FilterAndProjectOperator.constructFilteredTsBlock; public class DeviceAttributeUpdater extends DeviceUpdater { @@ -152,13 +153,8 @@ private TsBlock getFilterTsBlock() { final List filterResultColumns = new ArrayList<>(Arrays.asList(curBlock.getValueColumns())); - // CASE evaluation can clear a common expression after the filter used it. Recompute it before - // materializing the column for projection within this same batch. - commonTransformerList.forEach( - columnTransformer -> { - columnTransformer.tryEvaluate(); - filterResultColumns.add(columnTransformer.getColumn()); - }); + appendCommonExpressionColumns( + filterResultColumns, commonTransformerList, curFilterColumn, deviceSchemaBatch.size()); final ColumnBuilder[] columnBuilders = filterTsBlockBuilder.getValueColumnBuilders(); final int rowCount = diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DevicePredicateHandlerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DevicePredicateHandlerTest.java index 08b56188b2125..149062fa53e48 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DevicePredicateHandlerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DevicePredicateHandlerTest.java @@ -25,6 +25,7 @@ import org.apache.iotdb.calc.transformation.dag.column.leaf.ConstantColumnTransformer; import org.apache.iotdb.calc.transformation.dag.column.leaf.IdentityColumnTransformer; import org.apache.iotdb.calc.transformation.dag.column.leaf.LeafColumnTransformer; +import org.apache.iotdb.calc.transformation.dag.column.unary.scalar.UpperColumnTransformer; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.schema.node.role.IDeviceMNode; import org.apache.iotdb.commons.schema.table.TsTable; @@ -164,10 +165,10 @@ public void testAttributeUpdaterDoesNotReuseProjectionAcrossBatches() throws Exc @Test public void testAttributeUpdaterRecomputesCommonExpressionClearedByCase() throws Exception { Graph filter = new Graph(); - ColumnTransformer shared = filter.input(0); + ColumnTransformer shared = new UpperColumnTransformer(STRING, reference(filter.input(0))); ColumnTransformer predicate = filter.caseWhen( - filter.equal(shared, filter.constant("match")), + filter.equal(shared, filter.constant("MATCH")), filter.constant(true), filter.constant(false)); Graph projection = new Graph(); @@ -183,7 +184,7 @@ public void testAttributeUpdaterRecomputesCommonExpressionClearedByCase() throws updateBatch(updater, "match"); updateBatch(updater, "other", "match"); assertEquals( - Arrays.asList(Arrays.asList("match", "match"), Arrays.asList("match", "match")), updates); + Arrays.asList(Arrays.asList("match", "MATCH"), Arrays.asList("match", "MATCH")), updates); } } From aecaa05e84d4ab3bc794c6b2df1cb1abb559d793 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Wed, 14 Oct 2026 12:39:10 +0800 Subject: [PATCH 8/8] Document cache follow-ups and preserve schema constructor compatibility Track the known DIFF failure with a correct-result reproducer. Guard common-expression evaluation counts, document helper side effects and shallow array accounting, and retain constructor descriptors used by precompiled subclasses. --- .../process/FilterAndProjectOperator.java | 11 +++ .../process/FilterAndProjectOperatorTest.java | 80 +++++++++++++++++++ .../schema/source/DevicePredicateHandler.java | 17 ++++ .../operator/schema/source/DeviceUpdater.java | 19 +++++ 4 files changed, 127 insertions(+) diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperator.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperator.java index b258ee71ebf1f..5307cde136075 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperator.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperator.java @@ -220,6 +220,15 @@ private TsBlock getFilterTsBlock(TsBlock input) { /** * Materializes common expressions for projection after filtering. CASE may have invalidated their * caches, so any recomputation must preserve the filter mask and skip rejected input rows. + * + *

This mutates both the destination list and transformer cache state: uncached expressions are + * evaluated with selection, which can replace or clear descendant caches, and each appended + * column is consumed through {@link ColumnTransformer#getColumn()}, decrementing its cache + * reference count and possibly releasing the cached reference. Existing cached columns are + * reused; this method does not reset the DAG for a new input block. + * + *

Appended columns retain the original input row indexes. The caller must use the same filter + * to construct the compacted output block. */ public static void appendCommonExpressionColumns( List resultColumns, @@ -477,6 +486,8 @@ private int getMaxLevelOfColumnTransformerTree(ColumnTransformer columnTransform @Override public long ramBytesUsed() { + // Shallow array sizes include the header and every reference slot. This estimate does not + // include the transformer objects referenced by the arrays. return INSTANCE_SIZE + MemoryEstimationHelper.getEstimatedSizeOfAccountableObject(inputOperator) + MemoryEstimationHelper.getEstimatedSizeOfAccountableObject(operatorContext) diff --git a/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperatorTest.java b/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperatorTest.java index 0ba2c4d7ef50c..c9ac3ad328f41 100644 --- a/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperatorTest.java +++ b/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperatorTest.java @@ -36,6 +36,7 @@ import org.apache.iotdb.calc.transformation.dag.column.multi.MappableUDFColumnTransformer; import org.apache.iotdb.calc.transformation.dag.column.ternary.BetweenColumnTransformer; import org.apache.iotdb.calc.transformation.dag.column.unary.scalar.CastFunctionColumnTransformer; +import org.apache.iotdb.calc.transformation.dag.column.unary.scalar.DiffFunctionColumnTransformer; import org.apache.iotdb.calc.transformation.dag.udf.UDTFExecutor; import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNodeId; @@ -50,6 +51,7 @@ import org.apache.tsfile.read.common.block.column.RunLengthEncodedColumn; import org.apache.tsfile.read.common.block.column.TimeColumn; import org.apache.tsfile.utils.Binary; +import org.junit.Ignore; import org.junit.Test; import java.time.ZoneId; @@ -175,6 +177,62 @@ public void testFilterCommonExpressionsDoNotCrossBlocks() throws Exception { } } + @Test + public void testProjectionReadsMaterializedCommonExpression() throws Exception { + for (boolean withCase : new boolean[] {false, true}) { + Graph filter = new Graph(); + CountingAdditionColumnTransformer common = + new CountingAdditionColumnTransformer( + reference(filter.input(0)), reference(filter.constant(1))); + ColumnTransformer predicate = filter.greaterThan(common, 0); + if (withCase) { + predicate = + filter.greaterThan( + filter.caseWhen(predicate, filter.constant(1), filter.constant(0)), 0); + } + Graph projection = new Graph(); + try (Operator operator = + operator( + new TsBlock[] {block(-2, 3), block(4, -5)}, + filter, + predicate, + Collections.singletonList(common), + projection, + projection.input(1))) { + assertBlock(operator.next(), new Double[] {4.0}); + assertEquals(1, common.fullEvaluationCount); + assertEquals(withCase ? 1 : 0, common.selectedEvaluationCount); + assertBlock(operator.next(), new Double[] {5.0}); + assertEquals(2, common.fullEvaluationCount); + assertEquals(withCase ? 2 : 0, common.selectedEvaluationCount); + } + } + } + + @Ignore("Known incorrect DIFF state advancement: https://github.com/apache/iotdb/issues/18863") + @Test + public void testStatefulDiffIsNotReevaluatedAfterCaseFilter() throws Exception { + Graph filter = new Graph(); + ColumnTransformer diff = + new DiffFunctionColumnTransformer(DOUBLE, reference(filter.input(0)), true); + ColumnTransformer predicate = + filter.greaterThan( + filter.caseWhen(filter.greaterThan(diff, 1), filter.constant(1), filter.constant(0)), + 0); + Graph projection = new Graph(); + try (Operator operator = + operator( + new TsBlock[] {block(1, 3, 6), block(10, 11)}, + filter, + predicate, + Collections.singletonList(diff), + projection, + projection.input(1))) { + assertBlock(operator.next(), new Double[] {2.0, 3.0}); + assertBlock(operator.next(), new Double[] {4.0}); + } + } + @Test public void testCommonCastOnlyEvaluatesRowsPassingCaseFilter() throws Exception { for (boolean runLengthEncodedTime : new boolean[] {false, true}) { @@ -623,6 +681,28 @@ private ColumnTransformer caseWhen( } } + private static class CountingAdditionColumnTransformer + extends ArithmeticAdditionColumnTransformer { + private int fullEvaluationCount; + private int selectedEvaluationCount; + + private CountingAdditionColumnTransformer(ColumnTransformer left, ColumnTransformer right) { + super(DOUBLE, left, right); + } + + @Override + public void evaluate() { + fullEvaluationCount++; + super.evaluate(); + } + + @Override + public void evaluateWithSelection(boolean[] selection) { + selectedEvaluationCount++; + super.evaluateWithSelection(selection); + } + } + private static class CountingIdentityColumnTransformer extends IdentityColumnTransformer { private int evaluationCount; private int childTraversalCount; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DevicePredicateHandler.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DevicePredicateHandler.java index dd933d8403e75..b3cf75f120cc6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DevicePredicateHandler.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DevicePredicateHandler.java @@ -35,6 +35,7 @@ import org.apache.tsfile.read.common.block.TsBlockBuilder; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Objects; import java.util.stream.Collectors; @@ -60,6 +61,22 @@ public abstract class DevicePredicateHandler implements AutoCloseable { protected TsBlock curBlock; protected Column curFilterColumn; + // Retain the original constructor descriptor for precompiled subclasses without common roots. + protected DevicePredicateHandler( + final List filterLeafColumnTransformerList, + final ColumnTransformer filterOutputTransformer, + final List columnSchemaList, + final String database, + final TsTable table) { + this( + filterLeafColumnTransformerList, + filterOutputTransformer, + Collections.emptyList(), + columnSchemaList, + database, + table); + } + protected DevicePredicateHandler( final List filterLeafColumnTransformerList, final ColumnTransformer filterOutputTransformer, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DeviceUpdater.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DeviceUpdater.java index 5debad67608e3..c6690634e91cc 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DeviceUpdater.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DeviceUpdater.java @@ -31,6 +31,7 @@ import org.apache.tsfile.utils.Binary; +import java.util.Collections; import java.util.List; import java.util.function.BiFunction; @@ -38,6 +39,24 @@ public abstract class DeviceUpdater extends DevicePredicateHandler { private final BiFunction attributeProvider; + // Retain the original constructor descriptor for precompiled subclasses without common roots. + protected DeviceUpdater( + final List filterLeafColumnTransformerList, + final ColumnTransformer filterOutputTransformer, + final List columnSchemaList, + final BiFunction attributeProvider, + final String database, + final TsTable table) { + this( + filterLeafColumnTransformerList, + filterOutputTransformer, + Collections.emptyList(), + columnSchemaList, + attributeProvider, + database, + table); + } + protected DeviceUpdater( final List filterLeafColumnTransformerList, final ColumnTransformer filterOutputTransformer,