diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBCaseWhenThenIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBCaseWhenThenIT.java index a88b89e90b5da..7b3aacca9f0fa 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBCaseWhenThenIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBCaseWhenThenIT.java @@ -64,6 +64,13 @@ public class IoTDBCaseWhenThenIT { "INSERT INTO root.sg.d2(timestamp,s4) values(1000000, 55)", "INSERT INTO root.sg.d2(timestamp,s4) values(20000000, 66)", "INSERT INTO root.sg.d2(timestamp,s4) values(210000000, 77)", + "CREATE TIMESERIES root.sg.case_udf.x WITH DATATYPE=DOUBLE, ENCODING=PLAIN", + "CREATE TIMESERIES root.sg.case_udf.txt WITH DATATYPE=TEXT, ENCODING=PLAIN", + "CREATE TIMESERIES root.sg.case_udf.selector WITH DATATYPE=INT32, ENCODING=PLAIN", + "INSERT INTO root.sg.case_udf(time, x, txt, selector) VALUES " + + "(1, -1.0, 'sKiP', 0), (2, -2.0, 'mIxEd', 1), " + + "(3, -3.0, 'ThIrD', 2), (5, -5.0, 'lAsT', 0)", + "INSERT INTO root.sg.case_udf(time, selector) VALUES (4, 3)", }; @BeforeClass @@ -928,4 +935,75 @@ public void testThenWithBinarySameConstant() { new String[] {"0,0.0,", "1000000,11.0,", "20000000,22.0,", "210000000,33.0,"}; resultSetEqualTest(sql, expectedHeader, retArray); } + + @Test + public void testNullableAndInLaterCaseBranch() { + prepareData( + new String[] { + "CREATE TIMESERIES root.sg.case_null.s1 WITH DATATYPE=INT32, ENCODING=PLAIN", + "CREATE TIMESERIES root.sg.case_null.s2 WITH DATATYPE=INT32, ENCODING=PLAIN", + "INSERT INTO root.sg.case_null(time, s2) VALUES (1, 1)", + "INSERT INTO root.sg.case_null(time, s1, s2) VALUES (2, 1, 1), (3, 0, 1), (4, 101, 1)" + }); + resultSetEqualTest( + "SELECT CASE WHEN s1 > 100 THEN 1 WHEN s1 > 0 AND s2 > 0 THEN 2 ELSE 3 END AS result " + + "FROM root.sg.case_null", + new String[] {TIMESTAMP_STR, "result"}, + new String[] {"1,3.0,", "2,2.0,", "3,3.0,", "4,1.0,"}); + } + + @Test + public void testMappableUdfInCaseMultipleSelectedRows() { + resultSetEqualTest( + "SELECT CASE WHEN selector > 0 THEN abs(x) ELSE 0.0 END AS abs_result, " + + "CASE WHEN selector > 0 THEN upper(txt) ELSE 'skipped' END AS upper_result, " + + "CASE WHEN selector > 0 THEN lower(txt) ELSE 'skipped' END AS lower_result " + + "FROM root.sg.case_udf", + new String[] {TIMESTAMP_STR, "abs_result", "upper_result", "lower_result"}, + new String[] { + "1,0.0,skipped,skipped,", + "2,2.0,MIXED,mixed,", + "3,3.0,THIRD,third,", + "4,null,null,null,", + "5,0.0,skipped,skipped," + }); + } + + @Test + public void testMappableUdfInCaseSingleNonFirstSelectedRow() { + resultSetEqualTest( + "SELECT CASE WHEN selector = 2 THEN abs(x) ELSE 0.0 END AS abs_result, " + + "CASE WHEN selector = 2 THEN upper(txt) ELSE 'skipped' END AS upper_result, " + + "CASE WHEN selector = 2 THEN lower(txt) ELSE 'skipped' END AS lower_result " + + "FROM root.sg.case_udf", + new String[] {TIMESTAMP_STR, "abs_result", "upper_result", "lower_result"}, + new String[] { + "1,0.0,skipped,skipped,", + "2,0.0,skipped,skipped,", + "3,3.0,THIRD,third,", + "4,0.0,skipped,skipped,", + "5,0.0,skipped,skipped," + }); + } + + @Test + public void testMappableUdfInCaseWithCachedNullableInputs() { + // Project the raw inputs first so the CASE branches borrow their complete cached columns. + resultSetEqualTest( + "SELECT x AS raw_x, txt AS raw_txt, " + + "CASE WHEN selector > 0 THEN abs(x) ELSE 0.0 END AS abs_result, " + + "CASE WHEN selector > 0 THEN upper(txt) ELSE 'skipped' END AS upper_result, " + + "CASE WHEN selector > 0 THEN lower(txt) ELSE 'skipped' END AS lower_result " + + "FROM root.sg.case_udf", + new String[] { + TIMESTAMP_STR, "raw_x", "raw_txt", "abs_result", "upper_result", "lower_result" + }, + new String[] { + "1,-1.0,sKiP,0.0,skipped,skipped,", + "2,-2.0,mIxEd,2.0,MIXED,mixed,", + "3,-3.0,ThIrD,3.0,THIRD,third,", + "4,null,null,null,null,null,", + "5,-5.0,lAsT,0.0,skipped,skipped," + }); + } } diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBCaseWhenThenTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBCaseWhenThenTableIT.java index 50053867636bd..05ae9b227b3aa 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBCaseWhenThenTableIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBCaseWhenThenTableIT.java @@ -72,6 +72,7 @@ public class IoTDBCaseWhenThenTableIT { @BeforeClass public static void setUp() throws Exception { + EnvFactory.getEnv().getConfig().getCommonConfig().setMaxTsBlockLineNumber(1); EnvFactory.getEnv().initClusterEnvironment(); prepareTableData(SQLs); List moreSQLs = new ArrayList<>(); @@ -695,4 +696,213 @@ public void testThenWithBinarySameConstant() { String[] retArray = new String[] {"0,", "11,", "22,", "33,"}; tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE); } + + @Test + public void testCaseWhenReusesAggregationAcrossBlocks() { + prepareTableData( + new String[] { + "USE " + DATABASE, + "CREATE TABLE case_aggregation_cache (dev STRING TAG, c DOUBLE FIELD, q INT32 FIELD)", + "INSERT INTO case_aggregation_cache(time, dev, c, q) VALUES " + + "(0, 'd0', 1.0, 0), (0, 'd1', 1.0, 0), (3600000, 'd1', 1.0, 0)" + }); + + // The non-pushdown predicate and the different device/window shapes make the aggregation + // produce multiple blocks. CASE clears the shared aggregate before the next output uses it. + String suffix = " FROM case_aggregation_cache WHERE q % 2 < 1 GROUP BY dev, date_bin(1h, time)"; + for (String projection : + new String[] { + "CASE WHEN sum(c) > 0 THEN 1.0 END AS case_result, CAST(sum(c) AS DOUBLE) AS total", + "CASE WHEN sum(c) > 0 THEN 1.0 END AS case_result, sum(c) + 0 AS total", + "CASE WHEN sum(c) > 0 THEN CAST(sum(c) AS DOUBLE) / CAST(sum(c) AS DOUBLE) END " + + "AS case_result, CAST(sum(c) AS DOUBLE) AS total" + }) { + tableResultSetEqualTest( + "SELECT " + projection + suffix, + new String[] {"case_result", "total"}, + new String[] {"1.0,1.0,", "1.0,1.0,", "1.0,1.0,"}, + DATABASE); + } + } + + @Test + public void testCaseOperandsAndMultiArgumentFunctions() { + prepareTableData( + new String[] { + "USE " + DATABASE, + "CREATE TABLE case_operands (dev STRING TAG, x DOUBLE FIELD)", + "INSERT INTO case_operands(time, dev, x) VALUES (0, 'd0', 1.0), (1, 'd0', -2.0)" + }); + String caseWhen = "CASE WHEN x > 0 THEN 2.0 ELSE 3.0 END"; + String[] expressions = { + "x + " + caseWhen, + "CASE WHEN x > -100 THEN coalesce(x, 0.0) END", + "greatest(x, " + caseWhen + ")", + "CASE WHEN x BETWEEN -5.0 AND " + caseWhen + " THEN 1.0 ELSE 0.0 END", + "CASE WHEN x > -100 THEN CASE WHEN x BETWEEN -5.0 AND " + + caseWhen + + " THEN 1.0 ELSE 0.0 END END", + "coalesce(x, " + caseWhen + ")", + "CASE WHEN x > -100 THEN coalesce(x, " + caseWhen + ") END" + }; + String[][] results = { + {"3.0,", "1.0,"}, + {"1.0,", "-2.0,"}, + {"2.0,", "3.0,"}, + {"1.0,", "1.0,"}, + {"1.0,", "1.0,"}, + {"1.0,", "-2.0,"}, + {"1.0,", "-2.0,"} + }; + for (int i = 0; i < expressions.length; i++) { + tableResultSetEqualTest( + "SELECT " + expressions[i] + " AS result FROM case_operands ORDER BY time", + new String[] {"result"}, + results[i], + DATABASE); + } + tableResultSetEqualTest( + "SELECT coalesce(x, 0.0) AS result FROM case_operands " + + "WHERE CASE WHEN coalesce(x, 0.0) > 0 THEN 1.0 ELSE 0.0 END > 0", + new String[] {"result"}, + new String[] {"1.0,"}, + DATABASE); + } + + @Test + public void testCommonCastRespectsCaseFilter() { + prepareTableData( + new String[] { + "USE " + DATABASE, + "CREATE TABLE guarded_common_cast (dev STRING TAG, src STRING FIELD)", + "INSERT INTO guarded_common_cast(time, dev, src) VALUES " + + "(0, 'd0', 'n/a'), (1, 'd0', '5')" + }); + String query = + "SELECT CAST(src AS INT32) + 1 AS result FROM guarded_common_cast " + + "WHERE CASE WHEN src <> 'n/a' THEN CAST(src AS INT32) > 10 ELSE false END"; + tableResultSetEqualTest(query, new String[] {"result"}, new String[0], DATABASE); + + prepareTableData( + new String[] { + "USE " + DATABASE, + "INSERT INTO guarded_common_cast(time, dev, src) VALUES (2, 'd0', '50')" + }); + tableResultSetEqualTest(query, new String[] {"result"}, new String[] {"51,"}, DATABASE); + } + + @Test + public void testReusingCasePreservesPendingSiblingColumn() { + prepareTableData( + new String[] { + "USE " + DATABASE, + "CREATE TABLE case_sibling_cache (dev STRING TAG, x DOUBLE FIELD)", + "INSERT INTO case_sibling_cache(time, dev, x) VALUES " + + "(0, 'd0', -3.0), (1, 'd0', 0.0), (2, 'd0', 2.0)" + }); + + String sharedCase = "CASE WHEN x > 0 THEN 1.0 ELSE 0.0 END"; + tableResultSetEqualTest( + "SELECT " + + sharedCase + + " AS a, x + CASE WHEN (" + + sharedCase + + ") > 0 THEN 2.0 ELSE 3.0 END AS b FROM case_sibling_cache ORDER BY time", + new String[] {"a", "b"}, + new String[] {"0.0,0.0,", "0.0,3.0,", "1.0,4.0,"}, + DATABASE); + } + + @Test + public void testSharedDiffAcrossCaseFilterAndProjection() { + prepareTableData( + new String[] { + "USE " + DATABASE, + "CREATE TABLE case_shared_diff (dev STRING TAG, x DOUBLE FIELD)", + "INSERT INTO case_shared_diff(time, dev, x) VALUES " + + "(1, 'd0', 1.0), (2, 'd0', 3.0), (3, 'd0', 6.0)", + "FLUSH", + "INSERT INTO case_shared_diff(time, dev, x) VALUES (4, 'd0', 10.0), (5, 'd0', 11.0)", + "FLUSH" + }); + tableResultSetEqualTest( + "SELECT time, diff(x) AS d FROM case_shared_diff " + + "WHERE CASE WHEN diff(x) > 1.0 THEN 1.0 ELSE 0.0 END > 0.0", + new String[] {"time", "d"}, + new String[] { + "1970-01-01T00:00:00.002Z,2.0,", + "1970-01-01T00:00:00.003Z,3.0,", + "1970-01-01T00:00:00.004Z,4.0," + }, + DATABASE); + tableResultSetEqualTest( + "SELECT diff(x) AS d FROM case_shared_diff " + + "WHERE CASE WHEN diff(x) > 1.0 THEN true ELSE true END", + new String[] {"d"}, + new String[] {"null,", "2.0,", "3.0,", "4.0,", "1.0,"}, + DATABASE); + tableResultSetEqualTest( + "SELECT CASE WHEN x > 5 THEN diff(x) ELSE 100.0 END AS a, " + + "CASE WHEN x > 5 THEN 200.0 ELSE diff(x) END AS b, diff(x) AS d " + + "FROM case_shared_diff", + new String[] {"a", "b", "d"}, + new String[] { + "100.0,null,null,", "100.0,2.0,2.0,", "3.0,200.0,3.0,", "4.0,200.0,4.0,", "1.0,200.0,1.0," + }, + DATABASE); + } + + @Test + public void testDynamicIgnoreNullDiffAcrossCaseFilterAndProjection() { + prepareTableData( + new String[] { + "USE " + DATABASE, + "CREATE TABLE case_dynamic_diff " + + "(dev STRING TAG, x DOUBLE FIELD, ignore_null BOOLEAN FIELD)", + "INSERT INTO case_dynamic_diff(time, dev, x, ignore_null) VALUES " + + "(1, 'd0', 1.0, true), (2, 'd0', 3.0, false), (3, 'd0', null, true), " + + "(4, 'd0', 10.0, null), (5, 'd0', null, false)", + "FLUSH", + "INSERT INTO case_dynamic_diff(time, dev, x, ignore_null) VALUES " + + "(6, 'd0', 20.0, true), (7, 'd0', 25.0, false), " + + "(8, 'd0', null, true), (9, 'd0', 30.0, true)", + "FLUSH" + }); + String diff = "diff(x, ignore_null)"; + tableResultSetEqualTest( + "SELECT " + diff + " AS d FROM case_dynamic_diff ORDER BY time", + new String[] {"d"}, + new String[] {"null,", "2.0,", "null,", "7.0,", "null,", "null,", "5.0,", "null,", "5.0,"}, + DATABASE); + tableResultSetEqualTest( + "SELECT CASE WHEN x > 5 THEN " + + diff + + " ELSE 100.0 END AS a, CASE WHEN x > 5 THEN 200.0 ELSE " + + diff + + " END AS b, " + + diff + + " AS d FROM case_dynamic_diff ORDER BY time", + new String[] {"a", "b", "d"}, + new String[] { + "100.0,null,null,", + "100.0,2.0,2.0,", + "100.0,null,null,", + "7.0,200.0,7.0,", + "100.0,null,null,", + "null,200.0,null,", + "5.0,200.0,5.0,", + "100.0,null,null,", + "5.0,200.0,5.0," + }, + DATABASE); + tableResultSetEqualTest( + "SELECT " + + diff + + " AS d FROM case_dynamic_diff WHERE CASE WHEN x IS NULL THEN false WHEN " + + diff + + " > 1.0 THEN true ELSE false END ORDER BY time", + new String[] {"d"}, + new String[] {"2.0,", "7.0,", "5.0,", "5.0,"}, + DATABASE); + } } diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBCaseDeviceCacheIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBCaseDeviceCacheIT.java new file mode 100644 index 0000000000000..6596f17f4274e --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBCaseDeviceCacheIT.java @@ -0,0 +1,179 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.schema; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; +import org.apache.iotdb.itbase.env.BaseEnv; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; + +import static org.apache.iotdb.db.it.utils.TestUtils.assertResultSetEqual; +import static org.apache.iotdb.db.it.utils.TestUtils.prepareTableData; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBCaseDeviceCacheIT { + private static final String DATABASE = "case_device_cache"; + private static final String CASE_RESULT = "CASE WHEN src = 'match' THEN 'match' ELSE 'other' END"; + + @BeforeClass + public static void setUp() throws Exception { + // Keep every device in the same schema region and force a new batch for each device. + EnvFactory.getEnv().getConfig().getCommonConfig().setDefaultSchemaRegionGroupNumPerDatabase(1); + EnvFactory.getEnv().getConfig().getCommonConfig().setMaxTsBlockLineNumber(1); + EnvFactory.getEnv().initClusterEnvironment(); + prepareTableData(new String[] {"CREATE DATABASE " + DATABASE}); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void testDevicePredicatesAcrossBatches() throws Exception { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + createTable(statement, "predicate_cache"); + for (String predicate : new String[] {CASE_RESULT + " = src", "src = " + CASE_RESULT}) { + assertResultSetEqual( + statement.executeQuery("SHOW DEVICES FROM predicate_cache WHERE " + predicate), + "id,src,a,b,", + new HashSet<>( + Arrays.asList( + "d0,match,null,null,", + "d1,match,null,null,", + "d2,other,null,null,", + "d3,other,null,null,"))); + assertResultSetEqual( + statement.executeQuery("COUNT DEVICES FROM predicate_cache WHERE " + predicate), + "count(devices),", + Collections.singleton("4,")); + } + } + } + + @Test + public void testAttributeProjectionAcrossBatches() throws Exception { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + createTable(statement, "projection_cache"); + statement.execute("UPDATE projection_cache SET a = " + CASE_RESULT + ", b = src"); + assertResultSetEqual( + statement.executeQuery("SHOW DEVICES FROM projection_cache"), + "id,src,a,b,", + new HashSet<>( + Arrays.asList( + "d0,match,match,match,", + "d1,match,match,match,", + "d2,other,other,other,", + "d3,other,other,other,"))); + statement.execute( + "UPDATE projection_cache SET a = " + + "concat(src, CASE WHEN src = 'match' THEN '-m' ELSE '-o' END)"); + assertResultSetEqual( + statement.executeQuery("SHOW DEVICES FROM projection_cache"), + "id,src,a,b,", + new HashSet<>( + Arrays.asList( + "d0,match,match-m,match,", + "d1,match,match-m,match,", + "d2,other,other-o,other,", + "d3,other,other-o,other,"))); + } + } + + @Test + public void testAttributeCommonExpressionClearedByCase() throws Exception { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + createTable(statement, "common_cache"); + statement.execute( + "UPDATE common_cache SET a = upper(src) " + + "WHERE CASE WHEN upper(src) = 'MATCH' THEN true ELSE false END"); + assertResultSetEqual( + statement.executeQuery("SHOW DEVICES FROM common_cache"), + "id,src,a,b,", + new HashSet<>( + Arrays.asList( + "d0,match,MATCH,null,", + "d1,match,MATCH,null,", + "d2,other,null,null,", + "d3,other,null,null,"))); + } + } + + @Test + public void testGuardedCommonCastInAttributeUpdate() throws Exception { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("USE " + DATABASE); + statement.execute( + "CREATE TABLE guarded_cast " + + "(id STRING TAG, src STRING ATTRIBUTE, a STRING ATTRIBUTE, v DOUBLE FIELD)"); + statement.execute( + "INSERT INTO guarded_cast(time, id, src, a, v) VALUES " + + "(1, 'd0', 'n/a', 'seed', 1), (2, 'd1', '5', 'seed', 1)"); + String update = + "UPDATE guarded_cast SET a = CAST(CAST(src AS INT32) + 1 AS STRING) " + + "WHERE CASE WHEN src <> 'n/a' THEN CAST(src AS INT32) > 10 ELSE false END"; + + statement.execute(update); + assertResultSetEqual( + statement.executeQuery("SHOW DEVICES FROM guarded_cast"), + "id,src,a,", + new HashSet<>(Arrays.asList("d0,n/a,seed,", "d1,5,seed,"))); + + statement.execute("UPDATE guarded_cast SET src = '50' WHERE id = 'd1'"); + statement.execute(update); + assertResultSetEqual( + statement.executeQuery("SHOW DEVICES FROM guarded_cast"), + "id,src,a,", + new HashSet<>(Arrays.asList("d0,n/a,seed,", "d1,50,51,"))); + } + } + + private static void createTable(Statement statement, String table) throws SQLException { + statement.execute("USE " + DATABASE); + statement.execute( + "CREATE TABLE " + + table + + " (id STRING TAG, src STRING ATTRIBUTE, a STRING ATTRIBUTE, b STRING ATTRIBUTE, v DOUBLE FIELD)"); + statement.execute( + "INSERT INTO " + + table + + "(time, id, src, v) VALUES (1, 'd0', 'match', 1), (2, 'd1', 'match', 1), " + + "(3, 'd2', 'other', 1), (4, 'd3', 'other', 1)"); + } +} diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperator.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperator.java index 67d0d69874a36..cf352b680c20b 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperator.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperator.java @@ -32,6 +32,8 @@ import org.apache.iotdb.calc.transformation.dag.column.multi.MultiColumnTransformer; import org.apache.iotdb.calc.transformation.dag.column.ternary.TernaryColumnTransformer; import org.apache.iotdb.calc.transformation.dag.column.unary.UnaryColumnTransformer; +import org.apache.iotdb.calc.transformation.dag.column.unary.scalar.DiffColumnTransformer; +import org.apache.iotdb.calc.transformation.dag.column.unary.scalar.DiffFunctionColumnTransformer; import org.apache.iotdb.commons.queryengine.execution.MemoryEstimationHelper; import com.google.common.util.concurrent.ListenableFuture; @@ -41,20 +43,30 @@ import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.read.common.block.TsBlock; import org.apache.tsfile.read.common.block.TsBlockBuilder; +import org.apache.tsfile.read.common.block.column.DoubleColumn; import org.apache.tsfile.read.common.block.column.RunLengthEncodedColumn; import org.apache.tsfile.read.common.block.column.TimeColumnBuilder; import org.apache.tsfile.utils.Pair; import org.apache.tsfile.utils.RamUsageEstimator; import java.util.ArrayList; +import java.util.Collections; +import java.util.IdentityHashMap; import java.util.List; import java.util.OptionalInt; +import java.util.Set; + +import static org.apache.tsfile.utils.RamUsageEstimator.sizeOfBooleanArray; +import static org.apache.tsfile.utils.RamUsageEstimator.sizeOfDoubleArray; public class FilterAndProjectOperator implements ProcessOperator { private static final long INSTANCE_SIZE = RamUsageEstimator.shallowSizeOfInstance(FilterAndProjectOperator.class); + private static final long DIFF_COLUMN_INSTANCE_SIZE = + RamUsageEstimator.shallowSizeOfInstance(DoubleColumn.class); + private final Operator inputOperator; private final List filterLeafColumnTransformerList; @@ -67,6 +79,12 @@ public class FilterAndProjectOperator implements ProcessOperator { private final List projectOutputTransformerList; + private final ColumnTransformer[] filterCacheTransformers; + + private final ColumnTransformer[] projectCacheTransformers; + + private final int retainedInputColumnCount; + private final TsBlockBuilder filterTsBlockBuilder; private final boolean hasNonMappableUDF; @@ -100,6 +118,15 @@ public FilterAndProjectOperator( this.hasNonMappableUDF = hasNonMappableUDF; this.filterTsBlockBuilder = new TsBlockBuilder(8, filterOutputDataTypes); this.hasFilter = hasFilter; + List filterRoots = new ArrayList<>(commonTransformerList); + if (filterOutputTransformer != null) { + filterRoots.add(filterOutputTransformer); + } + this.filterCacheTransformers = ColumnTransformer.collectTransformers(filterRoots); + this.projectCacheTransformers = + ColumnTransformer.collectTransformers(projectOutputTransformerList); + this.retainedInputColumnCount = + countRetainedInputColumns(filterCacheTransformers, projectCacheTransformers); } public FilterAndProjectOperator( @@ -111,12 +138,28 @@ public FilterAndProjectOperator( this.projectLeafColumnTransformerList = filterAndProjectOperator.projectLeafColumnTransformerList; this.projectOutputTransformerList = filterAndProjectOperator.projectOutputTransformerList; + this.filterCacheTransformers = filterAndProjectOperator.filterCacheTransformers; + this.projectCacheTransformers = filterAndProjectOperator.projectCacheTransformers; + this.retainedInputColumnCount = filterAndProjectOperator.retainedInputColumnCount; this.hasNonMappableUDF = filterAndProjectOperator.hasNonMappableUDF; this.hasFilter = filterAndProjectOperator.hasFilter; this.filterTsBlockBuilder = filterAndProjectOperator.filterTsBlockBuilder; this.inputOperator = inputOperator; } + private static int countRetainedInputColumns(ColumnTransformer[]... phases) { + Set retained = Collections.newSetFromMap(new IdentityHashMap<>()); + for (ColumnTransformer[] transformers : phases) { + for (ColumnTransformer transformer : transformers) { + if (transformer instanceof DiffFunctionColumnTransformer + || transformer instanceof DiffColumnTransformer) { + retained.add(transformer); + } + } + } + return retained.size(); + } + @Override public CommonOperatorContext getOperatorContext() { return operatorContext; @@ -134,7 +177,13 @@ public TsBlock next() throws Exception { } long inputRowCount = input.getPositionCount(); - TsBlock filterResult = getFilterTsBlock(input); + TsBlock filterResult; + try { + filterResult = getFilterTsBlock(input); + } finally { + // Materialization and compaction have consumed every filter result before projection starts. + clearColumnCaches(filterCacheTransformers); + } filteredRowCount += inputRowCount - filterResult.getPositionCount(); operatorContext.recordSpecifiedInfo("Filtered Rows", Long.toString(filteredRowCount)); @@ -145,6 +194,12 @@ public TsBlock next() throws Exception { return getTransformedTsBlock(filterResult); } + private static void clearColumnCaches(ColumnTransformer[] transformers) { + for (ColumnTransformer transformer : transformers) { + transformer.clearColumnCache(); + } + } + /** * Return the TsBlock that contains both initial input columns and columns of common * subexpressions after filtering. @@ -152,6 +207,12 @@ public TsBlock next() throws Exception { private TsBlock getFilterTsBlock(TsBlock input) { final Column originTimeColumn = input.getTimeColumn(); final int positionCount = originTimeColumn.getPositionCount(); + // A selected consumer does not consume full-cache references. Release all results from the + // preceding input, including stateful transformers' per-input results, before feeding this + // block. + for (ColumnTransformer columnTransformer : filterCacheTransformers) { + columnTransformer.clearCacheForNewInput(); + } // feed Filter ColumnTransformer, including TimeStampColumnTransformer and constant for (LeafColumnTransformer leafColumnTransformer : filterLeafColumnTransformerList) { leafColumnTransformer.initFromTsBlock(input); @@ -172,13 +233,8 @@ private TsBlock getFilterTsBlock(TsBlock input) { } if (!hasNonMappableUDF) { - // get result of calculated common sub expressions - for (ColumnTransformer columnTransformer : commonTransformerList) { - // CASE WHEN clause would clear all its cache - // evaluate again to acquire cache - columnTransformer.tryEvaluate(); - resultColumns.add(columnTransformer.getColumn()); - } + appendCommonExpressionColumns( + resultColumns, commonTransformerList, filterColumn, positionCount); } final ColumnBuilder[] columnBuilders = filterTsBlockBuilder.getValueColumnBuilders(); @@ -205,6 +261,24 @@ private TsBlock getFilterTsBlock(TsBlock input) { } } + // Selected results belong to this materialization, not the shared full-column caches. + private static void appendCommonExpressionColumns( + List resultColumns, + List commonTransformers, + Column filterColumn, + int positionCount) { + if (commonTransformers.isEmpty()) { + return; + } + boolean[] selection = new boolean[positionCount]; + for (int i = 0; i < positionCount; i++) { + selection[i] = satisfy(filterColumn, i); + } + for (ColumnTransformer transformer : commonTransformers) { + resultColumns.add(transformer.getColumnWithSelection(selection)); + } + } + private static int constructFilteredTsBlock( List resultColumns, TimeColumnBuilder timeBuilder, @@ -265,18 +339,28 @@ public static boolean satisfy(Column filterColumn, int rowIndex) { private TsBlock getTransformedTsBlock(TsBlock input) { final Column originTimeColumn = input.getTimeColumn(); final int positionCount = originTimeColumn.getPositionCount(); + // The flat array was deduplicated at construction; clear each node before evaluating any + // output. + for (ColumnTransformer columnTransformer : projectCacheTransformers) { + columnTransformer.clearCacheForNewInput(); + } // feed pre calculated data for (LeafColumnTransformer leafColumnTransformer : projectLeafColumnTransformerList) { leafColumnTransformer.initFromTsBlock(input); } - List resultColumns = new ArrayList<>(projectOutputTransformerList.size()); - for (ColumnTransformer columnTransformer : projectOutputTransformerList) { - columnTransformer.tryEvaluate(); - resultColumns.add(columnTransformer.getColumn()); + try { + List resultColumns = new ArrayList<>(projectOutputTransformerList.size()); + for (ColumnTransformer columnTransformer : projectOutputTransformerList) { + columnTransformer.tryEvaluate(); + resultColumns.add(columnTransformer.getColumn()); + } + return TsBlock.wrapBlocksWithoutCopy( + positionCount, originTimeColumn, resultColumns.toArray(new Column[0])); + } finally { + // Returned columns are retained by the TsBlock, not by ordinary transformer caches. + clearColumnCaches(projectCacheTransformers); } - return TsBlock.wrapBlocksWithoutCopy( - positionCount, originTimeColumn, resultColumns.toArray(new Column[0])); } @Override @@ -319,7 +403,7 @@ public long calculateMaxPeekMemory() { maxPeekMemory, (long) maxCachedColumn * TSFileDescriptor.getInstance().getConfig().getPageSizeInByte()) - + inputOperator.calculateRetainedSizeAfterCallingNext(); + + calculateRetainedSizeAfterCallingNext(); } // has Filter @@ -336,7 +420,7 @@ public long calculateMaxPeekMemory() { return Math.max( maxPeekMemory, (long) maxCachedColumn * TSFileDescriptor.getInstance().getConfig().getPageSizeInByte()) - + inputOperator.calculateRetainedSizeAfterCallingNext(); + + calculateRetainedSizeAfterCallingNext(); } @Override @@ -353,7 +437,20 @@ public long calculateMaxReturnSize() { @Override public long calculateRetainedSizeAfterCallingNext() { - return inputOperator.calculateRetainedSizeAfterCallingNext(); + // DIFF retains a complete result until the next input even after its ordinary cache is + // consumed. + // Filter and projection results can coexist, so count distinct nodes across both phases. + return inputOperator.calculateRetainedSizeAfterCallingNext() + + (long) retainedInputColumnCount * getMaxRetainedDiffColumnSize(); + } + + private static long getMaxRetainedDiffColumnSize() { + int maxRows = TSFileDescriptor.getInstance().getConfig().getMaxTsBlockLineNumber(); + // DIFF always builds a DoubleColumn. A larger TsBlock can exceed a storage page, so account + // for the value array, NULL flags, and object/array headers using the column's sizing rules. + return Math.max( + TSFileDescriptor.getInstance().getConfig().getPageSizeInByte(), + DIFF_COLUMN_INSTANCE_SIZE + sizeOfBooleanArray(maxRows) + sizeOfDoubleArray(maxRows)); } private int getMaxLevelOfColumnTransformerTree(ColumnTransformer columnTransformer) { @@ -438,9 +535,13 @@ private int getMaxLevelOfColumnTransformerTree(ColumnTransformer columnTransform @Override public long ramBytesUsed() { + // Shallow array sizes include the header and every reference slot. This estimate does not + // include the transformer objects referenced by the arrays. return INSTANCE_SIZE + MemoryEstimationHelper.getEstimatedSizeOfAccountableObject(inputOperator) + MemoryEstimationHelper.getEstimatedSizeOfAccountableObject(operatorContext) + + RamUsageEstimator.shallowSizeOf(filterCacheTransformers) + + RamUsageEstimator.shallowSizeOf(projectCacheTransformers) + filterTsBlockBuilder.getRetainedSizeInBytes(); } } diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/AbstractCaseWhenThenColumnTransformer.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/AbstractCaseWhenThenColumnTransformer.java index 8452a590a25a1..b20cd12cf5290 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/AbstractCaseWhenThenColumnTransformer.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/AbstractCaseWhenThenColumnTransformer.java @@ -30,6 +30,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.function.Consumer; public abstract class AbstractCaseWhenThenColumnTransformer extends ColumnTransformer { @@ -68,7 +69,7 @@ public void evaluate() { // region evaluate first when ColumnTransformer firstWhenColumnTransformer = whenThenTransformers.get(0).left; - firstWhenColumnTransformer.evaluate(); + firstWhenColumnTransformer.tryEvaluate(); Column firstWhenColumn = firstWhenColumnTransformer.getColumn(); int positionCount = firstWhenColumn.getPositionCount(); @@ -96,8 +97,7 @@ public void evaluate() { } ColumnTransformer firstThenColumnTransformer = whenThenTransformers.get(0).right; - firstThenColumnTransformer.evaluateWithSelection(selectionForThen); - Column firstThenColumn = firstThenColumnTransformer.getColumn(); + Column firstThenColumn = firstThenColumnTransformer.getColumnWithSelection(selectionForThen); thenColumnList.add(firstThenColumn); // endregion @@ -105,8 +105,7 @@ public void evaluate() { // when and then columns for (int i = 1; i < whenThenTransformers.size(); i++) { ColumnTransformer whenColumnTransformer = whenThenTransformers.get(i).left; - whenColumnTransformer.evaluateWithSelection(selection); - Column whenColumn = whenColumnTransformer.getColumn(); + Column whenColumn = whenColumnTransformer.getColumnWithSelection(selection); selectionForThen = selection.clone(); @@ -122,17 +121,16 @@ public void evaluate() { } ColumnTransformer thenColumnTransformer = whenThenTransformers.get(i).right; - thenColumnTransformer.evaluateWithSelection(selectionForThen); - Column thenColumn = thenColumnTransformer.getColumn(); + Column thenColumn = thenColumnTransformer.getColumnWithSelection(selectionForThen); thenColumnList.add(thenColumn); } // elseColumn - doTransform(branchIndexForEachRow, positionCount, thenColumnList); + initializeColumnCache(doTransform(branchIndexForEachRow, positionCount, thenColumnList)); } @Override - public void evaluateWithSelection(boolean[] selection) { + protected Column evaluateSelection(boolean[] selection) { // region initialize branchIndexForEachRow. // branchIndexForEachRow indicates the index of the WhenTransformer matched by each row. @@ -157,8 +155,7 @@ public void evaluateWithSelection(boolean[] selection) { // when and then columns for (int i = 0; i < whenThenTransformers.size(); i++) { ColumnTransformer whenColumnTransformer = whenThenTransformers.get(i).left; - whenColumnTransformer.evaluateWithSelection(selectionForWhen); - Column whenColumn = whenColumnTransformer.getColumn(); + Column whenColumn = whenColumnTransformer.getColumnWithSelection(selectionForWhen); boolean[] selectionForThen = selectionForWhen.clone(); @@ -174,17 +171,16 @@ public void evaluateWithSelection(boolean[] selection) { } ColumnTransformer thenColumnTransformer = whenThenTransformers.get(i).right; - thenColumnTransformer.evaluateWithSelection(selectionForThen); - Column thenColumn = thenColumnTransformer.getColumn(); + Column thenColumn = thenColumnTransformer.getColumnWithSelection(selectionForThen); thenColumnList.add(thenColumn); } // elseColumn // when selectionForElse[i] is false represent the rows that do not need to evaluate - doTransform(branchIndexForEachRow, positionCount, thenColumnList); + return doTransform(branchIndexForEachRow, positionCount, thenColumnList); } - private void doTransform( + private Column doTransform( int[] branchIndexForEachRow, int positionCount, List thenColumnList) { boolean[] selectionForElse = new boolean[positionCount]; for (int i = 0; i < branchIndexForEachRow.length; i++) { @@ -192,10 +188,8 @@ private void doTransform( selectionForElse[i] = true; } } - elseTransformer.evaluateWithSelection(selectionForElse); - + Column elseColumn = elseTransformer.getColumnWithSelection(selectionForElse); ColumnBuilder builder = returnType.createColumnBuilder(positionCount); - Column elseColumn = elseTransformer.getColumn(); for (int i = 0; i < positionCount; i++) { Column resultColumn = null; @@ -215,15 +209,7 @@ private void doTransform( } } - initializeColumnCache(builder.build()); - // Because of short-circuit evaluation, CaseWhen does not calculate all Then phrases, so its - // calculation results cannot be reused. - for (Pair whenThenColumnTransformer : - whenThenTransformers) { - whenThenColumnTransformer.left.clearCache(); - whenThenColumnTransformer.right.clearCache(); - } - elseTransformer.clearCache(); + return builder.build(); } protected abstract void writeToColumnBuilder( @@ -234,6 +220,16 @@ protected void checkType() { // do nothing } + @Override + protected void forEachChild(Consumer action) { + for (Pair whenThenColumnTransformer : + whenThenTransformers) { + action.accept(whenThenColumnTransformer.left); + action.accept(whenThenColumnTransformer.right); + } + action.accept(elseTransformer); + } + @Override public void close() { super.close(); diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/ColumnCache.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/ColumnCache.java index 6ada45d6d10e4..c7cf716c1fcaf 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/ColumnCache.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/ColumnCache.java @@ -62,4 +62,8 @@ public void cacheColumn(Column column, int referenceCount) { public boolean hasCached() { return column != null; } + + public Column peekColumn() { + return column; + } } diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/ColumnTransformer.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/ColumnTransformer.java index d10d728cd2db5..af2319ff0a51d 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/ColumnTransformer.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/ColumnTransformer.java @@ -25,6 +25,13 @@ import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.read.common.type.TypeEnum; +import java.util.ArrayList; +import java.util.Collections; +import java.util.IdentityHashMap; +import java.util.List; +import java.util.Set; +import java.util.function.Consumer; + import static org.apache.iotdb.calc.plan.relational.metadata.CommonMetadataUtils.isNumericType; public abstract class ColumnTransformer { @@ -33,6 +40,9 @@ public abstract class ColumnTransformer { protected final ColumnCache columnCache; + // Compatibility with the evaluateWithSelection()/getColumn() API. Never used as a full cache. + private ColumnCache selectedColumnCache; + protected int referenceCount; protected ColumnTransformer(Type returnType) { @@ -42,24 +52,54 @@ protected ColumnTransformer(Type returnType) { } public void tryEvaluate() { + selectedColumnCache = null; if (!columnCache.hasCached()) { evaluate(); } } + /** + * @deprecated Use {@link #getColumnWithSelection(boolean[])} to retain the result directly. + */ + @Deprecated + public void tryEvaluateWithSelection(boolean[] selection) { + selectedColumnCache = null; + if (!columnCache.hasCached()) { + evaluateWithSelection(selection); + } + } + public Column getColumn() { + if (selectedColumnCache != null) { + return selectedColumnCache.getColumn(); + } return columnCache.getColumn(); } + /** + * Returns a column whose selected rows are evaluated, without consuming or replacing the shared + * full-column cache. Unselected rows are unspecified. Callers must retain the returned column and + * only read selected rows. Partial results belong to the caller, so different CASE branches can + * use different masks without invalidating one another's columns. + */ + public Column getColumnWithSelection(boolean[] selection) { + Column fullColumn = columnCache.peekColumn(); + return fullColumn != null ? fullColumn : evaluateSelection(selection); + } + public void addReferenceCount() { referenceCount++; } public void initializeColumnCache(Column column) { + selectedColumnCache = null; columnCache.cacheColumn(column, referenceCount); } public int getColumnCachePositionCount() { + if (selectedColumnCache != null) { + return selectedColumnCache.getPositionCount(); + } return columnCache.getPositionCount(); } @@ -104,15 +144,85 @@ public boolean typeNotEquals(TypeEnum typeEnum) { /** Responsible for the calculation. */ protected abstract void evaluate(); - public abstract void evaluateWithSelection(boolean[] selection); + /** + * @deprecated Use {@link #getColumnWithSelection(boolean[])} to retain the result directly. + */ + @Deprecated + public void evaluateWithSelection(boolean[] selection) { + Column result = getColumnWithSelection(selection); + selectedColumnCache = new ColumnCache(); + selectedColumnCache.cacheColumn(result, referenceCount); + } + + protected abstract Column evaluateSelection(boolean[] selection); protected abstract void checkType(); public void close() { - // do nothing + clearColumnCache(); } + /** + * Releases only this node's ordinary cached columns after all consumers in an evaluation phase + * finish. Stateful per-input results remain valid until {@link #clearCacheForNewInput()}. + */ + public final void clearColumnCache() { + selectedColumnCache = null; + columnCache.clear(); + } + + /** + * Clears ordinary column caches in this subgraph. Input-specific retained results, such as DIFF's + * complete result, remain valid so another request for the same input cannot advance state again. + * This method does not mark a new input: before replacing leaf inputs, call {@link + * #clearCacheForNewInput()} on every node collected by {@link #collectTransformers(Iterable)}. + * Subclasses should define child edges via {@link #forEachChild(Consumer)} instead of overriding + * this method. + * + *

Kept non-final so existing Mockito-based tests can stub it. + */ public void clearCache() { - this.columnCache.clear(); + clearColumnCache(); + forEachChild(ColumnTransformer::clearCache); + } + + /** + * Clears only this node's cached column before initializing a new input block. Call this on every + * node collected by {@link #collectTransformers(Iterable)} before evaluating any expressions for + * the new block, when no sibling expression can still need a cached column. + */ + public final void clearCacheForNewInput() { + clearColumnCache(); + resetForNewInput(); + } + + /** Releases input-specific state while preserving state carried between blocks, such as DIFF. */ + protected void resetForNewInput() { + // Stateless transformers only need their shared cache cleared. } + + /** + * Collects all nodes reachable from the evaluation phase's roots, including CASE descendants, + * deduplicating by identity. Collect once after constructing the DAG and retain the flat array so + * new-input cache clearing needs no graph traversal or visited-set allocation. + */ + public static ColumnTransformer[] collectTransformers(Iterable roots) { + List transformers = new ArrayList<>(); + Set visited = Collections.newSetFromMap(new IdentityHashMap<>()); + for (ColumnTransformer root : roots) { + root.collectTransformers(transformers, visited); + } + return transformers.toArray(new ColumnTransformer[0]); + } + + private void collectTransformers( + List transformers, Set visited) { + if (visited.add(this)) { + transformers.add(this); + forEachChild(child -> child.collectTransformers(transformers, visited)); + } + } + + /** Defines the child edges used for cache clearing and input lifecycle management. */ + protected abstract void forEachChild(Consumer action); } diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/FailFunctionColumnTransformer.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/FailFunctionColumnTransformer.java index 351c4396c0fc3..af52965ab742d 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/FailFunctionColumnTransformer.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/FailFunctionColumnTransformer.java @@ -21,9 +21,12 @@ import org.apache.iotdb.commons.exception.SemanticException; +import org.apache.tsfile.block.column.Column; import org.apache.tsfile.read.common.block.column.NullColumn; import org.apache.tsfile.read.common.type.Type; +import java.util.function.Consumer; + /** * when evaluate of FailFunctionColumnTransformer is called, throw Exception to inform the user with * specified errorMsg @@ -44,7 +47,7 @@ protected void evaluate() { } @Override - public void evaluateWithSelection(boolean[] selection) { + protected Column evaluateSelection(boolean[] selection) { // if there is true value in selection, throw exception for (boolean b : selection) { if (b) { @@ -52,11 +55,16 @@ public void evaluateWithSelection(boolean[] selection) { } } // Result of fail function should be ignored, we just fake the output here. - initializeColumnCache(new NullColumn(1)); + return new NullColumn(selection.length); } @Override protected void checkType() { // do nothing } + + @Override + protected void forEachChild(Consumer action) { + // The fail function has no children. + } } diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/binary/BinaryColumnTransformer.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/binary/BinaryColumnTransformer.java index 5403e9f995f13..4390626276761 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/binary/BinaryColumnTransformer.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/binary/BinaryColumnTransformer.java @@ -25,6 +25,8 @@ import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.read.common.type.Type; +import java.util.function.Consumer; + public abstract class BinaryColumnTransformer extends ColumnTransformer { protected final ColumnTransformer leftTransformer; @@ -42,10 +44,11 @@ protected BinaryColumnTransformer( @Override public void evaluate() { leftTransformer.tryEvaluate(); - rightTransformer.tryEvaluate(); // attention: get positionCount before calling getColumn int positionCount = leftTransformer.getColumnCachePositionCount(); Column leftColumn = leftTransformer.getColumn(); + // Retain each input column before evaluating the next sibling. + rightTransformer.tryEvaluate(); Column rightColumn = rightTransformer.getColumn(); ColumnBuilder builder = returnType.createColumnBuilder(positionCount); @@ -54,21 +57,14 @@ public void evaluate() { } @Override - public void evaluateWithSelection(boolean[] selection) { - leftTransformer.evaluateWithSelection(selection); - // attention: get positionCount before calling getColumn - int positionCount = leftTransformer.getColumnCachePositionCount(); - Column leftColumn = leftTransformer.getColumn(); - // When the rightTransformer has the same constant as the leftTransformer, the leftTransformer's - // cache will be cleared, so need to getColumn before evaluate rightTransformer. - rightTransformer.evaluateWithSelection(selection); - Column rightColumn = rightTransformer.getColumn(); + protected Column evaluateSelection(boolean[] selection) { + Column leftColumn = leftTransformer.getColumnWithSelection(selection); + Column rightColumn = rightTransformer.getColumnWithSelection(selection); + int positionCount = leftColumn.getPositionCount(); ColumnBuilder builder = returnType.createColumnBuilder(positionCount); doTransform(leftColumn, rightColumn, builder, positionCount, selection); - initializeColumnCache(builder.build()); - this.leftTransformer.clearCache(); - this.rightTransformer.clearCache(); + return builder.build(); } protected abstract void doTransform( @@ -90,10 +86,9 @@ public ColumnTransformer getRightTransformer() { } @Override - public void clearCache() { - super.clearCache(); - this.leftTransformer.clearCache(); - this.rightTransformer.clearCache(); + protected void forEachChild(Consumer action) { + action.accept(leftTransformer); + action.accept(rightTransformer); } @Override diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/binary/LogicAndColumnTransformer.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/binary/LogicAndColumnTransformer.java index 5e7197eaab893..3516784149137 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/binary/LogicAndColumnTransformer.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/binary/LogicAndColumnTransformer.java @@ -32,21 +32,19 @@ public LogicAndColumnTransformer( } @Override - public void evaluateWithSelection(boolean[] selection) { + protected Column evaluateSelection(boolean[] selection) { boolean[] selectionCopy = selection.clone(); - leftTransformer.evaluateWithSelection(selectionCopy); - int positionCount = leftTransformer.getColumnCachePositionCount(); - Column leftColumn = leftTransformer.getColumn(); + Column leftColumn = leftTransformer.getColumnWithSelection(selectionCopy); + int positionCount = leftColumn.getPositionCount(); for (int i = 0; i < positionCount; i++) { if (!leftColumn.isNull(i) && !leftColumn.getBoolean(i)) { selectionCopy[i] = false; } } - rightTransformer.evaluateWithSelection(selectionCopy); - Column rightColumn = rightTransformer.getColumn(); + Column rightColumn = rightTransformer.getColumnWithSelection(selectionCopy); ColumnBuilder builder = returnType.createColumnBuilder(positionCount); for (int i = 0; i < positionCount; i++) { @@ -77,9 +75,7 @@ public void evaluateWithSelection(boolean[] selection) { } } - initializeColumnCache(rightColumn); - this.leftTransformer.clearCache(); - this.rightTransformer.clearCache(); + return builder.build(); } @Override diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/binary/LogicOrColumnTransformer.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/binary/LogicOrColumnTransformer.java index 5f2437238b7f6..fa7ca37519a15 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/binary/LogicOrColumnTransformer.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/binary/LogicOrColumnTransformer.java @@ -32,21 +32,19 @@ public LogicOrColumnTransformer( } @Override - public void evaluateWithSelection(boolean[] selection) { + protected Column evaluateSelection(boolean[] selection) { boolean[] selectionCopy = selection.clone(); - leftTransformer.evaluateWithSelection(selectionCopy); - int positionCount = leftTransformer.getColumnCachePositionCount(); - Column leftColumn = leftTransformer.getColumn(); + Column leftColumn = leftTransformer.getColumnWithSelection(selectionCopy); + int positionCount = leftColumn.getPositionCount(); for (int i = 0; i < positionCount; i++) { if (!leftColumn.isNull(i) && leftColumn.getBoolean(i)) { selectionCopy[i] = false; } } - rightTransformer.evaluateWithSelection(selectionCopy); - Column rightColumn = rightTransformer.getColumn(); + Column rightColumn = rightTransformer.getColumnWithSelection(selectionCopy); ColumnBuilder builder = returnType.createColumnBuilder(positionCount); @@ -78,9 +76,7 @@ public void evaluateWithSelection(boolean[] selection) { } } - initializeColumnCache(builder.build()); - this.leftTransformer.clearCache(); - this.rightTransformer.clearCache(); + return builder.build(); } @Override diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/leaf/ConstantColumnTransformer.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/leaf/ConstantColumnTransformer.java index d16072d729838..aa919caf19acf 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/leaf/ConstantColumnTransformer.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/leaf/ConstantColumnTransformer.java @@ -39,7 +39,7 @@ protected void evaluate() { } @Override - public void evaluateWithSelection(boolean[] selection) { + protected Column evaluateSelection(boolean[] selection) { int positionCount = input.getPositionCount(); ColumnBuilder builder = returnType.createColumnBuilder(positionCount); for (int i = 0; i < positionCount; i++) { @@ -49,6 +49,6 @@ public void evaluateWithSelection(boolean[] selection) { builder.write(value, 0); } } - initializeColumnCache(builder.build()); + return builder.build(); } } diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/leaf/IdentityColumnTransformer.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/leaf/IdentityColumnTransformer.java index 38dfd063ac19c..14a03983f2158 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/leaf/IdentityColumnTransformer.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/leaf/IdentityColumnTransformer.java @@ -43,7 +43,7 @@ protected void evaluate() { } @Override - public void evaluateWithSelection(boolean[] selection) { + protected Column evaluateSelection(boolean[] selection) { ColumnBuilder builder = returnType.createColumnBuilder(selection.length); Column inputColumn = input.getColumn(inputIndex); for (int i = 0; i < selection.length; i++) { @@ -53,6 +53,6 @@ public void evaluateWithSelection(boolean[] selection) { builder.write(inputColumn, i); } } - initializeColumnCache(builder.build()); + return builder.build(); } } diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/leaf/LeafColumnTransformer.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/leaf/LeafColumnTransformer.java index e9f69657a5405..ce16518ce245d 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/leaf/LeafColumnTransformer.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/leaf/LeafColumnTransformer.java @@ -24,6 +24,8 @@ import org.apache.tsfile.read.common.block.TsBlock; import org.apache.tsfile.read.common.type.Type; +import java.util.function.Consumer; + public abstract class LeafColumnTransformer extends ColumnTransformer { protected TsBlock input; @@ -39,4 +41,9 @@ public void checkType() { public void initFromTsBlock(TsBlock input) { this.input = input; } + + @Override + protected void forEachChild(Consumer action) { + // Leaf transformers have no children. + } } diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/leaf/NullColumnTransformer.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/leaf/NullColumnTransformer.java index 31a01f67f3047..db1a2430371a0 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/leaf/NullColumnTransformer.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/leaf/NullColumnTransformer.java @@ -19,6 +19,7 @@ package org.apache.iotdb.calc.transformation.dag.column.leaf; +import org.apache.tsfile.block.column.Column; import org.apache.tsfile.read.common.block.column.NullColumn; import static org.apache.tsfile.read.common.type.UnknownType.UNKNOWN; @@ -35,7 +36,7 @@ protected void evaluate() { } @Override - public void evaluateWithSelection(boolean[] selection) { - initializeColumnCache(new NullColumn(input.getPositionCount())); + protected Column evaluateSelection(boolean[] selection) { + return new NullColumn(input.getPositionCount()); } } diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/leaf/TimeColumnTransformer.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/leaf/TimeColumnTransformer.java index 8d359a0904a6e..2e220ba2216bb 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/leaf/TimeColumnTransformer.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/leaf/TimeColumnTransformer.java @@ -36,7 +36,7 @@ protected void evaluate() { } @Override - public void evaluateWithSelection(boolean[] selection) { + protected Column evaluateSelection(boolean[] selection) { ColumnBuilder builder = TIMESTAMP.createColumnBuilder(selection.length); Column timeColumn = input.getTimeColumn(); for (int i = 0; i < selection.length; i++) { @@ -46,6 +46,6 @@ public void evaluateWithSelection(boolean[] selection) { builder.write(timeColumn, i); } } - initializeColumnCache(builder.build()); + return builder.build(); } } diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/LogicalAndMultiColumnTransformer.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/LogicalAndMultiColumnTransformer.java index 0510ce0c1944e..30d6e9d24acbb 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/LogicalAndMultiColumnTransformer.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/LogicalAndMultiColumnTransformer.java @@ -36,14 +36,13 @@ public LogicalAndMultiColumnTransformer( } @Override - public void evaluateWithSelection(boolean[] selection) { + protected Column evaluateSelection(boolean[] selection) { boolean[] selectionCopy = selection.clone(); boolean[] hasNull = new boolean[selection.length]; List childrenColumns = new ArrayList<>(columnTransformerList.size()); for (ColumnTransformer child : columnTransformerList) { - child.evaluateWithSelection(selectionCopy); - Column childColumn = child.getColumn(); + Column childColumn = child.getColumnWithSelection(selectionCopy); childrenColumns.add(childColumn); for (int i = 0; i < childColumn.getPositionCount(); i++) { if (childColumn.isNull(i)) { @@ -77,12 +76,7 @@ public void evaluateWithSelection(boolean[] selection) { } } - initializeColumnCache(builder.build()); - - // clear children's cache - for (ColumnTransformer child : columnTransformerList) { - child.clearCache(); - } + return builder.build(); } @Override diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/LogicalOrMultiColumnTransformer.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/LogicalOrMultiColumnTransformer.java index b754de9f13ab1..6f5c479ed98d0 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/LogicalOrMultiColumnTransformer.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/LogicalOrMultiColumnTransformer.java @@ -36,14 +36,13 @@ public LogicalOrMultiColumnTransformer( } @Override - public void evaluateWithSelection(boolean[] selection) { + protected Column evaluateSelection(boolean[] selection) { boolean[] selectionCopy = selection.clone(); boolean[] hasNull = new boolean[selection.length]; List childColumns = new ArrayList<>(columnTransformerList.size()); for (ColumnTransformer child : columnTransformerList) { - child.evaluateWithSelection(selectionCopy); - Column childColumn = child.getColumn(); + Column childColumn = child.getColumnWithSelection(selectionCopy); childColumns.add(childColumn); for (int i = 0; i < childColumn.getPositionCount(); i++) { @@ -77,11 +76,7 @@ public void evaluateWithSelection(boolean[] selection) { } } - initializeColumnCache(builder.build()); - - for (ColumnTransformer child : columnTransformerList) { - child.clearCache(); - } + return builder.build(); } @Override diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/MappableUDFColumnTransformer.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/MappableUDFColumnTransformer.java index f619f5692119c..3b545a2a44c89 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/MappableUDFColumnTransformer.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/MappableUDFColumnTransformer.java @@ -26,6 +26,8 @@ import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.read.common.type.Type; +import java.util.function.Consumer; + public class MappableUDFColumnTransformer extends ColumnTransformer { private final ColumnTransformer[] inputColumnTransformers; @@ -42,36 +44,67 @@ public MappableUDFColumnTransformer( @Override public void evaluate() { // pull columns from previous transformers - for (ColumnTransformer inputColumnTransformer : inputColumnTransformers) { - inputColumnTransformer.tryEvaluate(); + Column[] columns = new Column[inputColumnTransformers.length]; + for (int i = 0; i < inputColumnTransformers.length; i++) { + inputColumnTransformers[i].tryEvaluate(); + // Retain each input column before evaluating the next sibling. + columns[i] = inputColumnTransformers[i].getColumn(); } - doTransform(); + initializeColumnCache(doTransform(columns)); } @Override - public void evaluateWithSelection(boolean[] selection) { - for (ColumnTransformer inputColumnTransformer : inputColumnTransformers) { - inputColumnTransformer.evaluateWithSelection(selection); + protected Column evaluateSelection(boolean[] selection) { + int[] positions = new int[selection.length]; + int selectedCount = 0; + for (int i = 0; i < selection.length; i++) { + if (selection[i]) { + positions[selectedCount++] = i; + } } - doTransform(); - // clear cache - for (ColumnTransformer inputColumnTransformer : inputColumnTransformers) { - inputColumnTransformer.clearCache(); + Column[] columns = new Column[inputColumnTransformers.length]; + for (int i = 0; i < inputColumnTransformers.length; i++) { + ColumnTransformer inputTransformer = inputColumnTransformers[i]; + Column inputColumn = inputTransformer.getColumnWithSelection(selection); + // UDFs can read backing arrays directly. Materialize selected positions instead of passing + // dictionary or offset views whose arrays do not match their logical positions. + ColumnBuilder inputBuilder = inputTransformer.getType().createColumnBuilder(selectedCount); + for (int j = 0; j < selectedCount; j++) { + int position = positions[j]; + if (inputColumn.isNull(position)) { + inputBuilder.appendNull(); + } else { + inputBuilder.write(inputColumn, position); + } + } + columns[i] = inputBuilder.build(); + } + ColumnBuilder builder = returnType.createColumnBuilder(selection.length); + // A mappable UDF only sees selected rows, even when its inputs reuse full cached columns. + Column result = selectedCount == 0 ? null : doTransform(columns); + int resultIndex = 0; + for (boolean selected : selection) { + if (selected) { + if (result.isNull(resultIndex)) { + builder.appendNull(); + } else { + builder.write(result, resultIndex); + } + resultIndex++; + } else { + builder.appendNull(); + } } + return builder.build(); } - private void doTransform() { - int size = inputColumnTransformers.length; - Column[] columns = new Column[size]; - for (int i = 0; i < size; i++) { - columns[i] = inputColumnTransformers[i].getColumn(); - } + private Column doTransform(Column[] columns) { // construct input TsBlock with columns int positionCount = columns[0].getPositionCount(); ColumnBuilder builder = returnType.createColumnBuilder(positionCount); // executor UDF and cache result executor.execute(columns, builder); - initializeColumnCache(builder.build()); + return builder.build(); } @Override @@ -94,10 +127,9 @@ public void close() { } @Override - public void clearCache() { - super.clearCache(); + protected void forEachChild(Consumer action) { for (ColumnTransformer columnTransformer : inputColumnTransformers) { - columnTransformer.clearCache(); + action.accept(columnTransformer); } } } diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/MultiColumnTransformer.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/MultiColumnTransformer.java index 827836a39ccb3..6d650c0d6b94a 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/MultiColumnTransformer.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/multi/MultiColumnTransformer.java @@ -27,7 +27,7 @@ import java.util.ArrayList; import java.util.List; -import java.util.stream.Collectors; +import java.util.function.Consumer; public abstract class MultiColumnTransformer extends ColumnTransformer { @@ -42,41 +42,34 @@ protected MultiColumnTransformer(Type returnType, List column @Override public void evaluate() { + List childrenColumns = new ArrayList<>(columnTransformerList.size()); for (ColumnTransformer child : columnTransformerList) { child.tryEvaluate(); + // Retain each input column before evaluating the next sibling. + childrenColumns.add(child.getColumn()); } - // attention: get positionCount before calling getColumn - int positionCount = columnTransformerList.get(0).getColumnCachePositionCount(); + int positionCount = childrenColumns.get(0).getPositionCount(); ColumnBuilder builder = returnType.createColumnBuilder(positionCount); - doTransform( - columnTransformerList.stream() - .map(ColumnTransformer::getColumn) - .collect(Collectors.toList()), - builder, - positionCount); + doTransform(childrenColumns, builder, positionCount); initializeColumnCache(builder.build()); } @Override - public void evaluateWithSelection(boolean[] selection) { + protected Column evaluateSelection(boolean[] selection) { List childrenColumns = new ArrayList<>(columnTransformerList.size()); for (ColumnTransformer child : columnTransformerList) { - child.evaluateWithSelection(selection); - childrenColumns.add(child.getColumn()); + childrenColumns.add(child.getColumnWithSelection(selection)); } - int positionCount = columnTransformerList.get(0).getColumnCachePositionCount(); + // getColumn() can release the first child's cache; read the retained column's size instead. + int positionCount = childrenColumns.get(0).getPositionCount(); ColumnBuilder builder = returnType.createColumnBuilder(positionCount); doTransform(childrenColumns, builder, positionCount, selection); - initializeColumnCache(builder.build()); - - for (ColumnTransformer child : columnTransformerList) { - child.clearCache(); - } + return builder.build(); } protected abstract void doTransform( @@ -90,11 +83,8 @@ public List getChildren() { } @Override - public void clearCache() { - super.clearCache(); - for (ColumnTransformer columnTransformer : columnTransformerList) { - columnTransformer.clearCache(); - } + protected void forEachChild(Consumer action) { + columnTransformerList.forEach(action); } @Override diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/ternary/TernaryColumnTransformer.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/ternary/TernaryColumnTransformer.java index 51648f797e3f7..03c56d7d90900 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/ternary/TernaryColumnTransformer.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/ternary/TernaryColumnTransformer.java @@ -25,6 +25,8 @@ import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.read.common.type.Type; +import java.util.function.Consumer; + public abstract class TernaryColumnTransformer extends ColumnTransformer { protected ColumnTransformer firstColumnTransformer; @@ -48,11 +50,12 @@ protected TernaryColumnTransformer( @Override public void evaluate() { firstColumnTransformer.tryEvaluate(); - secondColumnTransformer.tryEvaluate(); - thirdColumnTransformer.tryEvaluate(); int positionCount = firstColumnTransformer.getColumnCachePositionCount(); Column firstColumn = firstColumnTransformer.getColumn(); + // Retain each input column before evaluating the next sibling. + secondColumnTransformer.tryEvaluate(); Column secondColumn = secondColumnTransformer.getColumn(); + thirdColumnTransformer.tryEvaluate(); Column thirdColumn = thirdColumnTransformer.getColumn(); ColumnBuilder columnBuilder = returnType.createColumnBuilder(positionCount); doTransform(firstColumn, secondColumn, thirdColumn, columnBuilder, positionCount); @@ -60,20 +63,14 @@ public void evaluate() { } @Override - public void evaluateWithSelection(boolean[] selection) { - firstColumnTransformer.evaluateWithSelection(selection); - secondColumnTransformer.evaluateWithSelection(selection); - thirdColumnTransformer.evaluateWithSelection(selection); - int positionCount = firstColumnTransformer.getColumnCachePositionCount(); - Column firstColumn = firstColumnTransformer.getColumn(); - Column secondColumn = secondColumnTransformer.getColumn(); - Column thirdColumn = thirdColumnTransformer.getColumn(); + protected Column evaluateSelection(boolean[] selection) { + Column firstColumn = firstColumnTransformer.getColumnWithSelection(selection); + Column secondColumn = secondColumnTransformer.getColumnWithSelection(selection); + Column thirdColumn = thirdColumnTransformer.getColumnWithSelection(selection); + int positionCount = firstColumn.getPositionCount(); ColumnBuilder columnBuilder = returnType.createColumnBuilder(positionCount); doTransform(firstColumn, secondColumn, thirdColumn, columnBuilder, positionCount, selection); - initializeColumnCache(columnBuilder.build()); - firstColumnTransformer.clearCache(); - secondColumnTransformer.clearCache(); - thirdColumnTransformer.clearCache(); + return columnBuilder.build(); } protected abstract void doTransform( @@ -104,11 +101,10 @@ public ColumnTransformer getThirdColumnTransformer() { } @Override - public void clearCache() { - super.clearCache(); - firstColumnTransformer.clearCache(); - secondColumnTransformer.clearCache(); - thirdColumnTransformer.clearCache(); + protected void forEachChild(Consumer action) { + action.accept(firstColumnTransformer); + action.accept(secondColumnTransformer); + action.accept(thirdColumnTransformer); } @Override diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/udf/UserDefineScalarFunctionTransformer.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/udf/UserDefineScalarFunctionTransformer.java index 6ef156a6b707c..cddc145fb5ec9 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/udf/UserDefineScalarFunctionTransformer.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/udf/UserDefineScalarFunctionTransformer.java @@ -112,7 +112,7 @@ protected void doTransform( while (iterator.hasNext()) { try { Record input = iterator.next(); - if (selection[i++]) { + if (!selection[i++]) { builder.appendNull(); continue; } diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/unary/UnaryColumnTransformer.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/unary/UnaryColumnTransformer.java index 2a88aea340a95..496e8e8494e4c 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/unary/UnaryColumnTransformer.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/unary/UnaryColumnTransformer.java @@ -25,6 +25,8 @@ import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.read.common.type.Type; +import java.util.function.Consumer; + public abstract class UnaryColumnTransformer extends ColumnTransformer { protected ColumnTransformer childColumnTransformer; @@ -44,13 +46,11 @@ public void evaluate() { } @Override - public void evaluateWithSelection(boolean[] selection) { - childColumnTransformer.evaluateWithSelection(selection); - Column column = childColumnTransformer.getColumn(); + protected Column evaluateSelection(boolean[] selection) { + Column column = childColumnTransformer.getColumnWithSelection(selection); ColumnBuilder columnBuilder = returnType.createColumnBuilder(column.getPositionCount()); doTransform(column, columnBuilder, selection); - initializeColumnCache(columnBuilder.build()); - childColumnTransformer.clearCache(); + return columnBuilder.build(); } public ColumnTransformer getChildColumnTransformer() { @@ -68,9 +68,8 @@ protected abstract void doTransform( Column column, ColumnBuilder columnBuilder, boolean[] selection); @Override - public void clearCache() { - super.clearCache(); - childColumnTransformer.clearCache(); + protected void forEachChild(Consumer action) { + action.accept(childColumnTransformer); } @Override diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/unary/scalar/DiffColumnTransformer.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/unary/scalar/DiffColumnTransformer.java index 5aa14fb6f245d..7437c119fdd3e 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/unary/scalar/DiffColumnTransformer.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/unary/scalar/DiffColumnTransformer.java @@ -34,6 +34,9 @@ public class DiffColumnTransformer extends BinaryColumnTransformer { // indicate whether lastValue is null private boolean lastValueIsNull = true; + // DIFF must advance its state once per input, even when several masks/consumers request it. + private Column currentInputResult; + public DiffColumnTransformer( Type returnType, ColumnTransformer leftTransformer, ColumnTransformer rightTransformer) { super(returnType, leftTransformer, rightTransformer); @@ -44,6 +47,42 @@ protected void checkType() { // do nothing } + @Override + public void evaluate() { + initializeColumnCache(evaluateCurrentInput()); + } + + @Override + protected Column evaluateSelection(boolean[] selection) { + // DIFF depends on all preceding rows and their ignoreNull flags, even outside the mask. + return evaluateCurrentInput(); + } + + private Column evaluateCurrentInput() { + if (currentInputResult == null) { + leftTransformer.tryEvaluate(); + Column leftColumn = leftTransformer.getColumn(); + rightTransformer.tryEvaluate(); + Column rightColumn = rightTransformer.getColumn(); + int positionCount = leftColumn.getPositionCount(); + ColumnBuilder builder = returnType.createColumnBuilder(positionCount); + doTransform(leftColumn, rightColumn, builder, positionCount); + currentInputResult = builder.build(); + } + return currentInputResult; + } + + @Override + protected void resetForNewInput() { + currentInputResult = null; + } + + @Override + public void close() { + currentInputResult = null; + super.close(); + } + @Override protected void doTransform( Column leftColumn, Column rightColumn, ColumnBuilder builder, int positionCount) { diff --git a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/unary/scalar/DiffFunctionColumnTransformer.java b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/unary/scalar/DiffFunctionColumnTransformer.java index 04d65c594b9c5..dfa9d0808c16f 100644 --- a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/unary/scalar/DiffFunctionColumnTransformer.java +++ b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/transformation/dag/column/unary/scalar/DiffFunctionColumnTransformer.java @@ -26,8 +26,6 @@ import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.read.common.type.Type; -import java.util.Arrays; - public class DiffFunctionColumnTransformer extends UnaryColumnTransformer { // default is true @@ -39,6 +37,9 @@ public class DiffFunctionColumnTransformer extends UnaryColumnTransformer { // indicate whether lastValue is null private boolean lastValueIsNull = true; + // DIFF must advance its state once per input, even when several masks/consumers request it. + private Column currentInputResult; + public DiffFunctionColumnTransformer( Type returnType, ColumnTransformer childColumnTransformer, boolean ignoreNull) { super(returnType, childColumnTransformer); @@ -46,16 +47,36 @@ public DiffFunctionColumnTransformer( } @Override - public void evaluateWithSelection(boolean[] selection) { - // The DIFF function depends on forward and backward rows and cannot be short-circuited. - boolean[] selectionCopy = new boolean[selection.length]; - Arrays.fill(selectionCopy, true); - childColumnTransformer.evaluateWithSelection(selectionCopy); - Column column = childColumnTransformer.getColumn(); - ColumnBuilder columnBuilder = returnType.createColumnBuilder(column.getPositionCount()); - doTransform(column, columnBuilder, selection); - initializeColumnCache(columnBuilder.build()); - childColumnTransformer.clearCache(); + public void evaluate() { + initializeColumnCache(evaluateCurrentInput()); + } + + @Override + protected Column evaluateSelection(boolean[] selection) { + // DIFF depends on all preceding rows, including rows rejected by a CASE/filter mask. + return evaluateCurrentInput(); + } + + private Column evaluateCurrentInput() { + if (currentInputResult == null) { + childColumnTransformer.tryEvaluate(); + Column column = childColumnTransformer.getColumn(); + ColumnBuilder builder = returnType.createColumnBuilder(column.getPositionCount()); + doTransform(column, builder); + currentInputResult = builder.build(); + } + return currentInputResult; + } + + @Override + protected void resetForNewInput() { + currentInputResult = null; + } + + @Override + public void close() { + currentInputResult = null; + super.close(); } @Override diff --git a/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperatorTest.java b/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperatorTest.java new file mode 100644 index 0000000000000..71f0251a210e2 --- /dev/null +++ b/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/execution/operator/process/FilterAndProjectOperatorTest.java @@ -0,0 +1,1004 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.calc.execution.operator.process; + +import org.apache.iotdb.calc.execution.operator.CommonOperatorContext; +import org.apache.iotdb.calc.execution.operator.Operator; +import org.apache.iotdb.calc.plan.planner.memory.MemoryReservationManager; +import org.apache.iotdb.calc.transformation.dag.column.ColumnTransformer; +import org.apache.iotdb.calc.transformation.dag.column.FailFunctionColumnTransformer; +import org.apache.iotdb.calc.transformation.dag.column.TableCaseWhenThenColumnTransformer; +import org.apache.iotdb.calc.transformation.dag.column.binary.ArithmeticAdditionColumnTransformer; +import org.apache.iotdb.calc.transformation.dag.column.binary.CompareGreaterThanColumnTransformer; +import org.apache.iotdb.calc.transformation.dag.column.binary.CompareNonEqualColumnTransformer; +import org.apache.iotdb.calc.transformation.dag.column.leaf.ConstantColumnTransformer; +import org.apache.iotdb.calc.transformation.dag.column.leaf.IdentityColumnTransformer; +import org.apache.iotdb.calc.transformation.dag.column.leaf.LeafColumnTransformer; +import org.apache.iotdb.calc.transformation.dag.column.leaf.NullColumnTransformer; +import org.apache.iotdb.calc.transformation.dag.column.multi.CoalesceColumnTransformer; +import org.apache.iotdb.calc.transformation.dag.column.multi.MappableUDFColumnTransformer; +import org.apache.iotdb.calc.transformation.dag.column.ternary.BetweenColumnTransformer; +import org.apache.iotdb.calc.transformation.dag.column.unary.scalar.CastFunctionColumnTransformer; +import org.apache.iotdb.calc.transformation.dag.column.unary.scalar.DiffColumnTransformer; +import org.apache.iotdb.calc.transformation.dag.column.unary.scalar.DiffFunctionColumnTransformer; +import org.apache.iotdb.calc.transformation.dag.udf.UDTFExecutor; +import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNodeId; + +import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.block.column.ColumnBuilder; +import org.apache.tsfile.common.conf.TSFileConfig; +import org.apache.tsfile.common.conf.TSFileDescriptor; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.block.TsBlock; +import org.apache.tsfile.read.common.block.column.BinaryColumn; +import org.apache.tsfile.read.common.block.column.BooleanColumn; +import org.apache.tsfile.read.common.block.column.DoubleColumn; +import org.apache.tsfile.read.common.block.column.RunLengthEncodedColumn; +import org.apache.tsfile.read.common.block.column.TimeColumn; +import org.apache.tsfile.utils.Binary; +import org.junit.Test; + +import java.time.ZoneId; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import java.util.function.Consumer; + +import static org.apache.tsfile.read.common.type.BooleanType.BOOLEAN; +import static org.apache.tsfile.read.common.type.DoubleType.DOUBLE; +import static org.apache.tsfile.read.common.type.IntType.INT32; +import static org.apache.tsfile.read.common.type.StringType.STRING; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +public class FilterAndProjectOperatorTest { + + @Test + public void testProjectionDoesNotReuseColumnsAcrossDifferentSizedBlocks() throws Exception { + Graph graph = new Graph(); + ColumnTransformer input = graph.input(0); + ColumnTransformer caseWhen = + graph.caseWhen(graph.greaterThan(input, 0), graph.constant(1), graph.nullValue()); + try (Operator operator = + project(graph, new TsBlock[] {block(1), block(-2, 3), block(4)}, caseWhen, input)) { + assertBlock(operator.next(), new Double[] {1.0}, new Double[] {1.0}); + assertBlock(operator.next(), new Double[] {null, 1.0}, new Double[] {-2.0, 3.0}); + assertBlock(operator.next(), new Double[] {1.0}, new Double[] {4.0}); + } + } + + @Test + public void testProjectionDoesNotReuseColumnsAcrossEqualSizedBlocks() throws Exception { + Graph graph = new Graph(); + ColumnTransformer input = graph.input(0); + ColumnTransformer caseWhen = + graph.caseWhen(graph.greaterThan(input, 0), graph.constant(1), graph.nullValue()); + try (Operator operator = + project(graph, new TsBlock[] {block(1, 2), block(-3, 4)}, caseWhen, input)) { + assertBlock(operator.next(), new Double[] {1.0, 1.0}, new Double[] {1.0, 2.0}); + assertBlock(operator.next(), new Double[] {null, 1.0}, new Double[] {-3.0, 4.0}); + } + } + + @Test + public void testProjectionHandlesEmptyBlocks() throws Exception { + Graph graph = new Graph(); + ColumnTransformer input = graph.input(0); + ColumnTransformer caseWhen = + graph.caseWhen(graph.greaterThan(input, 0), graph.constant(1), graph.nullValue()); + try (Operator operator = + project(graph, new TsBlock[] {block(1), block(), block(-2)}, caseWhen, input)) { + assertBlock(operator.next(), new Double[] {1.0}, new Double[] {1.0}); + assertBlock(operator.next(), new Double[0], new Double[0]); + assertBlock(operator.next(), new Double[] {null}, new Double[] {-2.0}); + } + } + + @Test + public void testProjectionClearsSharedIntermediateExpressions() throws Exception { + Graph graph = new Graph(); + ColumnTransformer shared = graph.add(graph.input(0), 0); + ColumnTransformer caseWhen = + graph.caseWhen(graph.greaterThan(shared, 0), graph.constant(1), graph.nullValue()); + try (Operator operator = + project(graph, new TsBlock[] {block(1), block(-2)}, caseWhen, graph.add(shared, 1))) { + assertBlock(operator.next(), new Double[] {1.0}, new Double[] {2.0}); + assertBlock(operator.next(), new Double[] {null}, new Double[] {-1.0}); + } + } + + @Test + public void testFilterDoesNotReuseColumnsAcrossBlocks() throws Exception { + Graph filter = new Graph(); + ColumnTransformer input = filter.input(0); + ColumnTransformer caseWhen = + filter.caseWhen(filter.greaterThan(input, 0), filter.constant(1), filter.constant(0)); + ColumnTransformer predicate = filter.greaterThan(filter.add(caseWhen, input), 0); + Graph projection = new Graph(); + try (Operator operator = + operator( + new TsBlock[] {block(1), block(-2, 3), block(-4)}, + filter, + predicate, + Collections.emptyList(), + projection, + projection.input(0))) { + assertBlock(operator.next(), new Double[] {1.0}); + assertBlock(operator.next(), new Double[] {3.0}); + assertBlock(operator.next(), new Double[0]); + } + } + + @Test + public void testFilterCommonExpressionsDoNotCrossBlocks() throws Exception { + for (boolean runLengthEncodedTime : new boolean[] {false, true}) { + Graph filter = new Graph(); + ColumnTransformer shared = filter.add(filter.input(0), 0); + ColumnTransformer caseWhen = + filter.caseWhen(filter.greaterThan(shared, 0), filter.constant(1), filter.constant(0)); + ColumnTransformer predicate = filter.greaterThan(caseWhen, 0); + Graph projection = new Graph(); + try (Operator operator = + operator( + new TsBlock[] { + block(runLengthEncodedTime, 1), + block(runLengthEncodedTime, -2, 3), + block(runLengthEncodedTime, 4, -5) + }, + filter, + predicate, + Collections.singletonList(shared), + projection, + projection.input(0), + projection.input(1))) { + assertBlock(operator.next(), new Double[] {1.0}, new Double[] {1.0}); + assertBlock(operator.next(), new Double[] {3.0}, new Double[] {3.0}); + assertBlock(operator.next(), new Double[] {4.0}, new Double[] {4.0}); + } + } + } + + @Test + public void testProjectionReadsMaterializedCommonExpression() throws Exception { + for (boolean withCase : new boolean[] {false, true}) { + Graph filter = new Graph(); + CountingAdditionColumnTransformer common = + new CountingAdditionColumnTransformer( + reference(filter.input(0)), reference(filter.constant(1))); + ColumnTransformer predicate = filter.greaterThan(common, 0); + if (withCase) { + predicate = + filter.greaterThan( + filter.caseWhen(predicate, filter.constant(1), filter.constant(0)), 0); + } + Graph projection = new Graph(); + try (Operator operator = + operator( + new TsBlock[] {block(-2, 3), block(4, -5)}, + filter, + predicate, + Collections.singletonList(common), + projection, + projection.input(1))) { + assertBlock(operator.next(), new Double[] {4.0}); + assertEquals(1, common.fullEvaluationCount); + assertEquals(0, common.selectedEvaluationCount); + assertBlock(operator.next(), new Double[] {5.0}); + assertEquals(2, common.fullEvaluationCount); + assertEquals(0, common.selectedEvaluationCount); + } + } + } + + @Test + public void testStatefulDiffIsNotReevaluatedAfterCaseFilter() throws Exception { + Graph filter = new Graph(); + ColumnTransformer diff = + new DiffFunctionColumnTransformer(DOUBLE, reference(filter.input(0)), true); + ColumnTransformer predicate = + filter.greaterThan( + filter.caseWhen(filter.greaterThan(diff, 1), filter.constant(1), filter.constant(0)), + 0); + Graph projection = new Graph(); + try (Operator operator = + operator( + new TsBlock[] {block(1, 3, 6), block(10, 11)}, + filter, + predicate, + Collections.singletonList(diff), + projection, + projection.input(1))) { + assertBlock(operator.next(), new Double[] {2.0, 3.0}); + assertBlock(operator.next(), new Double[] {4.0}); + } + } + + @Test + public void testCommonCastOnlyEvaluatesRowsPassingCaseFilter() throws Exception { + for (boolean runLengthEncodedTime : new boolean[] {false, true}) { + Graph filter = new Graph(); + ColumnTransformer input = filter.leaf(new IdentityColumnTransformer(STRING, 0)); + ColumnTransformer invalid = + filter.leaf( + new ConstantColumnTransformer( + STRING, + new BinaryColumn( + 1, + Optional.empty(), + new Binary[] {new Binary("n/a", TSFileConfig.STRING_CHARSET)}))); + ColumnTransformer guard = + new CompareNonEqualColumnTransformer(BOOLEAN, reference(input), reference(invalid)); + ColumnTransformer common = + new CastFunctionColumnTransformer(INT32, reference(input), ZoneId.of("UTC")); + ColumnTransformer otherwise = + filter.leaf( + new ConstantColumnTransformer( + BOOLEAN, new BooleanColumn(1, Optional.empty(), new boolean[] {false}))); + ColumnTransformer predicate = + new TableCaseWhenThenColumnTransformer( + BOOLEAN, + Collections.singletonList(reference(guard)), + Collections.singletonList(reference(filter.greaterThan(common, 10))), + reference(otherwise)); + IdentityColumnTransformer output = new IdentityColumnTransformer(INT32, 1); + reference(common); + try (Operator operator = + new FilterAndProjectOperator( + new TestOperatorContext(), + new ValuesOperator( + new TestOperatorContext(), + Arrays.asList( + stringBlock(runLengthEncodedTime, "n/a", "5"), + stringBlock(runLengthEncodedTime, "n/a", "50", "5"), + stringBlock(runLengthEncodedTime, "n/a", "60"))), + Arrays.asList(TSDataType.STRING, TSDataType.INT32), + filter.leaves, + reference(predicate), + Collections.singletonList(common), + Collections.singletonList(output), + Collections.singletonList(reference(output)), + false, + true)) { + assertEquals(0, operator.next().getPositionCount()); + TsBlock firstMatch = operator.next(); + assertEquals(1, firstMatch.getPositionCount()); + assertEquals(50, firstMatch.getColumn(0).getInt(0)); + TsBlock secondMatch = operator.next(); + assertEquals(1, secondMatch.getPositionCount()); + assertEquals(60, secondMatch.getColumn(0).getInt(0)); + } + } + } + + @Test + public void testCommonMultiArgumentExpressionRespectsCaseFilter() throws Exception { + Graph filter = new Graph(); + ColumnTransformer common = + new CoalesceColumnTransformer( + DOUBLE, Arrays.asList(reference(filter.input(0)), reference(filter.constant(0)))); + ColumnTransformer caseWhen = + filter.caseWhen(filter.greaterThan(common, 0), filter.constant(1), filter.constant(0)); + Graph projection = new Graph(); + try (Operator operator = + operator( + new TsBlock[] {block(-2, 3), block(4, -5)}, + filter, + filter.greaterThan(caseWhen, 0), + Collections.singletonList(common), + projection, + projection.input(1))) { + assertBlock(operator.next(), new Double[] {3.0}); + assertBlock(operator.next(), new Double[] {4.0}); + } + } + + @Test + public void testBinaryPreservesSiblingColumnBeforeCase() throws Exception { + Graph graph = new Graph(); + ColumnTransformer input = graph.input(0); + ColumnTransformer caseWhen = + graph.caseWhen(graph.greaterThan(input, 0), graph.constant(2), graph.constant(3)); + try (Operator operator = + project(graph, new TsBlock[] {block(1, -2)}, graph.add(input, caseWhen))) { + assertBlock(operator.next(), new Double[] {3.0, 1.0}); + } + } + + @Test + public void testMultiArgumentFunctionPreservesSiblingColumnBeforeCase() throws Exception { + for (boolean selected : new boolean[] {false, true}) { + Graph graph = new Graph(); + ColumnTransformer input = graph.input(0); + ColumnTransformer caseWhen = + graph.caseWhen(graph.greaterThan(input, 0), graph.constant(2), graph.constant(3)); + ColumnTransformer output = + new CoalesceColumnTransformer( + DOUBLE, Arrays.asList(reference(input), reference(caseWhen))); + if (selected) { + output = graph.caseWhen(graph.greaterThan(graph.constant(1), 0), output, graph.nullValue()); + } + try (Operator operator = project(graph, new TsBlock[] {block(1, -2)}, output)) { + assertBlock(operator.next(), new Double[] {1.0, -2.0}); + } + } + } + + @Test + public void testTernaryPreservesSiblingColumnsBeforeCase() throws Exception { + for (boolean selected : new boolean[] {false, true}) { + Graph graph = new Graph(); + ColumnTransformer input = graph.input(0); + ColumnTransformer caseWhen = + graph.caseWhen(graph.greaterThan(input, 0), graph.constant(2), graph.constant(3)); + ColumnTransformer between = + new BetweenColumnTransformer( + BOOLEAN, reference(input), reference(graph.constant(-5)), reference(caseWhen), false); + ColumnTransformer output = graph.caseWhen(between, graph.constant(1), graph.constant(0)); + if (selected) { + output = graph.caseWhen(graph.greaterThan(graph.constant(1), 0), output, graph.nullValue()); + } + try (Operator operator = project(graph, new TsBlock[] {block(1, -2)}, output)) { + assertBlock(operator.next(), new Double[] {1.0, 1.0}); + } + } + } + + @Test + public void testMappableUdfPreservesSiblingColumnsBeforeCase() throws Exception { + for (boolean selected : new boolean[] {false, true}) { + Graph graph = new Graph(); + ColumnTransformer input = graph.input(0); + ColumnTransformer caseWhen = + graph.caseWhen(graph.greaterThan(input, 0), graph.constant(2), graph.constant(3)); + ColumnTransformer output = sumUdf(input, caseWhen); + if (selected) { + output = graph.caseWhen(graph.greaterThan(graph.constant(1), 0), output, graph.nullValue()); + } + try (Operator operator = project(graph, new TsBlock[] {block(1, -2)}, output)) { + assertBlock(operator.next(), new Double[] {3.0, 1.0}); + } + } + } + + @Test + public void testSelectedMappableUdfSkipsRejectedRowsWithFullCachedInput() throws Exception { + Graph graph = new Graph(); + ColumnTransformer input = graph.input(0); + ColumnTransformer udf = + new MappableUDFColumnTransformer( + DOUBLE, + new ColumnTransformer[] {reference(input)}, + new UDTFExecutor("test", ZoneId.of("UTC")) { + @Override + public void execute(Column[] columns, ColumnBuilder builder) { + for (int i = 0; i < columns[0].getPositionCount(); i++) { + double value = columns[0].getDouble(i); + org.junit.Assert.assertTrue(value > 0); + builder.writeDouble(value + 1); + } + } + + @Override + public void beforeDestroy() {} + }); + ColumnTransformer output = graph.caseWhen(graph.greaterThan(input, 0), udf, graph.constant(0)); + try (Operator operator = + project(graph, new TsBlock[] {block(-2, 3), block(-4)}, input, output)) { + assertBlock(operator.next(), new Double[] {-2.0, 3.0}, new Double[] {0.0, 4.0}); + assertBlock(operator.next(), new Double[] {-4.0}, new Double[] {0.0}); + } + } + + @Test + public void testProjectionClearsCachedDescendantsOfNestedCase() throws Exception { + Graph graph = new Graph(); + ColumnTransformer shared = graph.add(graph.input(0), 0); + ColumnTransformer innerCase = + graph.caseWhen(graph.greaterThan(shared, 0), graph.constant(1), graph.constant(0)); + ColumnTransformer outerCase = + graph.caseWhen(graph.greaterThan(innerCase, 0), graph.constant(2), graph.constant(0)); + // Simulate an intermediate cache retained from the previous input block. Clearing only the + // outer CASE itself must not allow this cached value to reach the new block's WHEN condition. + shared.initializeColumnCache(doubleColumn(1)); + try (Operator operator = project(graph, new TsBlock[] {block(-2)}, outerCase)) { + assertBlock(operator.next(), new Double[] {0.0}); + } + } + + @Test + public void testThenExpressionIsRecomputedWithoutSelectionForLaterOutput() throws Exception { + Graph graph = new Graph(); + ColumnTransformer input = graph.input(0); + ColumnTransformer shared = graph.add(input, 1); + ColumnTransformer caseWhen = + graph.caseWhen(graph.greaterThan(input, 0), shared, graph.constant(0)); + try (Operator operator = + project(graph, new TsBlock[] {block(-2, 3), block(-4, 5)}, caseWhen, shared)) { + assertBlock(operator.next(), new Double[] {0.0, 4.0}, new Double[] {-1.0, 4.0}); + assertBlock(operator.next(), new Double[] {0.0, 6.0}, new Double[] {-3.0, 6.0}); + } + } + + @Test + public void testTwoCasesUseIndependentMasksForSharedExpression() throws Exception { + Graph graph = new Graph(); + ColumnTransformer input = graph.input(0); + ColumnTransformer shared = graph.add(input, 1); + ColumnTransformer positive = + graph.caseWhen(graph.greaterThan(input, 0), shared, graph.constant(100)); + ColumnTransformer negative = + graph.caseWhen(graph.greaterThan(input, 0), graph.constant(200), shared); + try (Operator operator = + project(graph, new TsBlock[] {block(-2, 3), block(4, -5)}, positive, negative, shared)) { + assertBlock( + operator.next(), + new Double[] {100.0, 4.0}, + new Double[] {-1.0, 200.0}, + new Double[] {-1.0, 4.0}); + assertBlock( + operator.next(), + new Double[] {5.0, 100.0}, + new Double[] {200.0, -4.0}, + new Double[] {5.0, -4.0}); + } + } + + @Test + public void testTwoCasesAndFullOutputShareStatefulDiff() throws Exception { + for (boolean reverseOutputs : new boolean[] {false, true}) { + Graph graph = new Graph(); + ColumnTransformer input = graph.input(0); + ColumnTransformer diff = new DiffFunctionColumnTransformer(DOUBLE, reference(input), true); + ColumnTransformer positive = + graph.caseWhen(graph.greaterThan(input, 5), diff, graph.constant(100)); + ColumnTransformer negative = + graph.caseWhen(graph.greaterThan(input, 5), graph.constant(200), diff); + ColumnTransformer[] outputs = + reverseOutputs + ? new ColumnTransformer[] {diff, positive, negative} + : new ColumnTransformer[] {positive, negative, diff}; + try (Operator operator = + project(graph, new TsBlock[] {block(1, 3, 6), block(10, 11)}, outputs)) { + Double[] firstDiff = {null, 2.0, 3.0}; + Double[] firstPositive = {100.0, 100.0, 3.0}; + Double[] firstNegative = {null, 2.0, 200.0}; + if (reverseOutputs) { + assertBlock(operator.next(), firstDiff, firstPositive, firstNegative); + assertBlock( + operator.next(), + new Double[] {4.0, 1.0}, + new Double[] {4.0, 1.0}, + new Double[] {200.0, 200.0}); + } else { + assertBlock(operator.next(), firstPositive, firstNegative, firstDiff); + assertBlock( + operator.next(), + new Double[] {4.0, 1.0}, + new Double[] {200.0, 200.0}, + new Double[] {4.0, 1.0}); + } + } + } + } + + @Test + public void testNestedAndIndependentDiffResultsAreIncludedInMemoryEstimates() throws Exception { + int diffCount = 8; + long pageSize = TSFileDescriptor.getInstance().getConfig().getPageSizeInByte(); + for (boolean nested : new boolean[] {true, false}) { + Graph graph = new Graph(); + ColumnTransformer input = graph.input(0); + ColumnTransformer output = nested ? input : graph.constant(0); + for (int i = 0; i < diffCount; i++) { + ColumnTransformer argument = nested ? output : graph.add(input, i); + ColumnTransformer diff = + new DiffFunctionColumnTransformer(DOUBLE, reference(argument), true); + output = nested ? diff : graph.add(output, diff); + } + try (Operator operator = project(graph, new TsBlock[] {block(1, 3, 6)}, output)) { + // Retained DIFF columns coexist with the input/scratch columns during evaluation. + assertTrue(operator.calculateMaxPeekMemory() >= (diffCount + 1) * pageSize); + operator.next(); + // The child has returned its only block, but every DIFF still retains its result. + assertEquals(diffCount * pageSize, operator.calculateRetainedSizeAfterCallingNext()); + } + } + } + + @Test + public void testFilterReleasesBorrowedColumnsBeforeProjection() throws Exception { + Graph filter = new Graph(); + ColumnTransformer shared = filter.add(filter.input(0), 1); + ColumnTransformer caseWhen = + filter.caseWhen(filter.greaterThan(shared, 0), shared, filter.constant(0)); + ColumnTransformer predicate = filter.greaterThan(caseWhen, 1); + Graph projection = new Graph(); + ColumnTransformer output = + projection.leaf( + new IdentityColumnTransformer(DOUBLE, 0) { + @Override + protected void evaluate() { + assertEquals(0, shared.getColumnCachePositionCount()); + super.evaluate(); + } + }); + try (Operator operator = + operator( + new TsBlock[] {block(1, 2, 3), block(-2, 0, 5)}, + filter, + predicate, + Collections.emptyList(), + projection, + output)) { + assertBlock(operator.next(), new Double[] {1.0, 2.0, 3.0}); + assertEquals(0, shared.getColumnCachePositionCount()); + assertBlock(operator.next(), new Double[] {5.0}); + assertEquals(0, shared.getColumnCachePositionCount()); + assertEquals(0, operator.calculateRetainedSizeAfterCallingNext()); + } + } + + @Test + public void testProjectionReleasesBorrowedColumnsAfterReturningOutput() throws Exception { + Graph graph = new Graph(); + ColumnTransformer shared = graph.add(graph.input(0), 1); + ColumnTransformer caseWhen = + graph.caseWhen(graph.greaterThan(shared, 0), shared, graph.constant(0)); + try (Operator operator = + project(graph, new TsBlock[] {block(1, 2, 3), block(-2, 0, 5)}, caseWhen)) { + TsBlock first = operator.next(); + assertBlock(first, new Double[] {2.0, 3.0, 4.0}); + assertEquals(0, shared.getColumnCachePositionCount()); + assertBlock(operator.next(), new Double[] {0.0, 1.0, 6.0}); + assertEquals(0, shared.getColumnCachePositionCount()); + assertEquals(0, operator.calculateRetainedSizeAfterCallingNext()); + // Releasing transformer caches does not invalidate columns already returned to callers. + assertBlock(first, new Double[] {2.0, 3.0, 4.0}); + } + } + + @Test + public void testLargeDiffColumnsAreIncludedInMemoryEstimates() throws Exception { + TSFileConfig config = TSFileDescriptor.getInstance().getConfig(); + int oldMaxRows = config.getMaxTsBlockLineNumber(); + int oldMaxBytes = config.getMaxTsBlockSizeInBytes(); + int rowCount = 20_000; + try { + config.setMaxTsBlockLineNumber(rowCount); + config.setMaxTsBlockSizeInBytes(1024 * 1024); + double[] values = new double[rowCount]; + for (int i = 0; i < rowCount; i++) { + values[i] = i; + } + boolean[] ignoreNull = new boolean[rowCount]; + Arrays.fill(ignoreNull, true); + TsBlock input = + new TsBlock( + new TimeColumn(rowCount, new long[rowCount]), + doubleColumn(values), + new BooleanColumn(rowCount, Optional.empty(), ignoreNull)); + Graph graph = new Graph(); + ColumnTransformer output = graph.input(0); + ColumnTransformer flags = graph.leaf(new IdentityColumnTransformer(BOOLEAN, 1)); + List diffs = new ArrayList<>(); + for (int i = 0; i < 8; i++) { + output = + i % 2 == 0 + ? new DiffFunctionColumnTransformer(DOUBLE, reference(output), true) + : new DiffColumnTransformer(DOUBLE, reference(output), reference(flags)); + diffs.add(output); + } + try (Operator operator = project(graph, new TsBlock[] {input}, output)) { + long estimatedPeak = operator.calculateMaxPeekMemory(); + TsBlock result = operator.next(); + assertEquals(rowCount, result.getPositionCount()); + long retainedBytes = 0; + boolean[] mask = new boolean[rowCount]; + for (ColumnTransformer diff : diffs) { + Column retained = diff.getColumnWithSelection(mask); + // Phase cleanup must retain DIFF's original result rather than advance state again. + assertTrue(retained.isNull(0)); + retainedBytes += retained.getRetainedSizeInBytes(); + } + assertTrue(retainedBytes > 8L * config.getPageSizeInByte()); + assertTrue(operator.calculateRetainedSizeAfterCallingNext() >= retainedBytes); + assertTrue(estimatedPeak >= retainedBytes + input.getRetainedSizeInBytes()); + } + } finally { + config.setMaxTsBlockLineNumber(oldMaxRows); + config.setMaxTsBlockSizeInBytes(oldMaxBytes); + } + } + + @Test + public void testDiffRetainedMemoryDeduplicatesSharedNodesAcrossPhases() throws Exception { + Graph filter = new Graph(); + ColumnTransformer input = filter.input(0); + ColumnTransformer unary = new DiffFunctionColumnTransformer(DOUBLE, reference(input), true); + ColumnTransformer flags = + filter.leaf( + new ConstantColumnTransformer( + BOOLEAN, new BooleanColumn(1, Optional.empty(), new boolean[] {true}))); + ColumnTransformer binary = + new DiffColumnTransformer(DOUBLE, reference(input), reference(flags)); + ColumnTransformer predicate = filter.greaterThan(filter.add(unary, binary), 0); + long pageSize = TSFileDescriptor.getInstance().getConfig().getPageSizeInByte(); + try (FilterAndProjectOperator original = + operator( + new TsBlock[0], + filter, + predicate, + Arrays.asList(unary, binary), + new Graph(), + unary, + binary, + unary); + FilterAndProjectOperator copy = + new FilterAndProjectOperator( + original, new ValuesOperator(new TestOperatorContext(), Collections.emptyList()))) { + assertEquals(2 * pageSize, original.calculateRetainedSizeAfterCallingNext()); + assertTrue(original.calculateMaxPeekMemory() >= 6 * pageSize); + assertEquals(2 * pageSize, copy.calculateRetainedSizeAfterCallingNext()); + assertEquals(original.calculateMaxPeekMemory(), copy.calculateMaxPeekMemory()); + } + } + + @Test + public void testCaseStillSkipsUnusedFailBranch() throws Exception { + Graph graph = new Graph(); + ColumnTransformer caseWhen = + graph.caseWhen( + graph.greaterThan(graph.input(0), 0), + graph.constant(1), + new FailFunctionColumnTransformer(DOUBLE, "The unused ELSE branch was evaluated")); + try (Operator operator = project(graph, new TsBlock[] {block(1, 2), block(3)}, caseWhen)) { + assertBlock(operator.next(), new Double[] {1.0, 1.0}); + assertBlock(operator.next(), new Double[] {1.0}); + } + } + + @Test + public void testProjectionReusesSharedExpressionsWithinEachBlock() throws Exception { + Graph graph = new Graph(); + CountingIdentityColumnTransformer input = new CountingIdentityColumnTransformer(); + graph.leaves.add(input); + ColumnTransformer shared = graph.add(input, 0); + try (Operator operator = + project(graph, new TsBlock[] {block(1), block(2)}, shared, graph.add(shared, 1))) { + assertBlock(operator.next(), new Double[] {1.0}, new Double[] {2.0}); + assertEquals(1, input.evaluationCount); + assertBlock(operator.next(), new Double[] {2.0}, new Double[] {3.0}); + assertEquals(2, input.evaluationCount); + } + } + + @Test + public void testCopiedOperatorReusesCollectedTransformers() throws Exception { + Graph graph = new Graph(); + CountingIdentityColumnTransformer input = new CountingIdentityColumnTransformer(); + graph.leaves.add(input); + try (FilterAndProjectOperator original = project(graph, new TsBlock[0], graph.add(input, 1)); + FilterAndProjectOperator copy = + new FilterAndProjectOperator( + original, + new ValuesOperator(new TestOperatorContext(), Arrays.asList(block(1), block(2))))) { + assertEquals(1, input.childTraversalCount); + assertBlock(copy.next(), new Double[] {2.0}); + assertBlock(copy.next(), new Double[] {3.0}); + assertEquals(1, input.childTraversalCount); + } + } + + @Test + public void testProjectionCollectsSharedDagOnlyOnce() throws Exception { + Graph graph = new Graph(); + CountingIdentityColumnTransformer input = new CountingIdentityColumnTransformer(); + graph.leaves.add(input); + ColumnTransformer shared = input; + // Each level references the same child twice. A traversal without deduplication would visit + // the input 4096 times per root, although this DAG contains only 13 distinct transformers. + for (int i = 0; i < 12; i++) { + shared = graph.add(shared, shared); + } + try (Operator operator = + project(graph, new TsBlock[] {block(1), block(2)}, shared, graph.add(shared, 1))) { + assertEquals(1, input.childTraversalCount); + assertBlock(operator.next(), new Double[] {4096.0}, new Double[] {4097.0}); + assertEquals(1, input.childTraversalCount); + assertBlock(operator.next(), new Double[] {8192.0}, new Double[] {8193.0}); + assertEquals(1, input.childTraversalCount); + } + } + + @Test + public void testFilterAndCommonExpressionsCollectSharedDagOnlyOnce() throws Exception { + Graph filter = new Graph(); + CountingIdentityColumnTransformer input = new CountingIdentityColumnTransformer(); + filter.leaves.add(input); + ColumnTransformer shared = filter.add(input, 1); + ColumnTransformer predicate = filter.greaterThan(shared, 0); + Graph projection = new Graph(); + try (Operator operator = + operator( + new TsBlock[] {block(1), block(-2, 3)}, + filter, + predicate, + Collections.singletonList(shared), + projection, + projection.input(0), + projection.input(1))) { + assertEquals(1, input.childTraversalCount); + assertBlock(operator.next(), new Double[] {1.0}, new Double[] {2.0}); + assertEquals(1, input.childTraversalCount); + assertBlock(operator.next(), new Double[] {3.0}, new Double[] {4.0}); + assertEquals(1, input.childTraversalCount); + } + } + + @Test + public void testMappableUdfClearsCaseDescendantsAcrossBlocks() throws Exception { + Graph graph = new Graph(); + ColumnTransformer input = graph.input(0); + ColumnTransformer caseWhen = + graph.caseWhen(graph.greaterThan(input, 0), graph.constant(1), graph.constant(0)); + ColumnTransformer udf = sumUdf(caseWhen, input); + try (Operator operator = + project(graph, new TsBlock[] {block(1), block(-2, 3), block(4)}, udf)) { + assertBlock(operator.next(), new Double[] {2.0}); + assertBlock(operator.next(), new Double[] {-2.0, 4.0}); + assertBlock(operator.next(), new Double[] {5.0}); + } + } + + private static ColumnTransformer sumUdf(ColumnTransformer left, ColumnTransformer right) { + return new MappableUDFColumnTransformer( + DOUBLE, + new ColumnTransformer[] {reference(left), reference(right)}, + new UDTFExecutor("test", ZoneId.of("UTC")) { + @Override + public void execute(Column[] columns, ColumnBuilder builder) { + for (int i = 0; i < columns[0].getPositionCount(); i++) { + builder.writeDouble(columns[0].getDouble(i) + columns[1].getDouble(i)); + } + } + + @Override + public void beforeDestroy() { + // No UDF resources are allocated by this test executor. + } + }); + } + + @Test + public void testReusingCasePreservesPendingSiblingColumn() throws Exception { + Graph graph = new Graph(); + ColumnTransformer input = graph.input(0); + ColumnTransformer sharedCase = + graph.caseWhen(graph.greaterThan(input, 0), graph.constant(1), graph.constant(0)); + ColumnTransformer nestedCase = + graph.caseWhen(graph.greaterThan(sharedCase, 0), graph.constant(2), graph.constant(3)); + // The first output leaves sharedCase cached. In the second output, input is evaluated before + // nestedCase reuses sharedCase. Its cleanup must preserve input until the addition reads it. + try (Operator operator = + project( + graph, + new TsBlock[] {block(-3, 0, 2), block(4, -1)}, + sharedCase, + graph.add(input, nestedCase))) { + assertBlock(operator.next(), new Double[] {0.0, 0.0, 1.0}, new Double[] {0.0, 3.0, 4.0}); + assertBlock(operator.next(), new Double[] {1.0, 0.0}, new Double[] {6.0, 2.0}); + } + } + + private static FilterAndProjectOperator project( + Graph graph, TsBlock[] blocks, ColumnTransformer... outputs) { + return operator(blocks, null, null, Collections.emptyList(), graph, outputs); + } + + private static FilterAndProjectOperator operator( + TsBlock[] blocks, + Graph filter, + ColumnTransformer predicate, + List common, + Graph projection, + ColumnTransformer... outputs) { + List projectOutputs = new ArrayList<>(); + for (ColumnTransformer output : outputs) { + projectOutputs.add(reference(output)); + } + for (ColumnTransformer transformer : common) { + reference(transformer); + } + return new FilterAndProjectOperator( + new TestOperatorContext(), + new ValuesOperator(new TestOperatorContext(), Arrays.asList(blocks)), + Collections.nCopies(1 + common.size(), TSDataType.DOUBLE), + filter == null ? Collections.emptyList() : filter.leaves, + predicate == null ? null : reference(predicate), + common, + projection.leaves, + projectOutputs, + false, + predicate != null); + } + + private static ColumnTransformer reference(ColumnTransformer transformer) { + transformer.addReferenceCount(); + return transformer; + } + + private static DoubleColumn doubleColumn(double... values) { + return new DoubleColumn(values.length, Optional.empty(), values); + } + + private static TsBlock block(double... values) { + return block(false, values); + } + + private static TsBlock stringBlock(boolean runLengthEncodedTime, String... values) { + Binary[] binaries = + Arrays.stream(values) + .map(value -> new Binary(value, TSFileConfig.STRING_CHARSET)) + .toArray(Binary[]::new); + return new TsBlock( + block(runLengthEncodedTime, new double[values.length]).getTimeColumn(), + new BinaryColumn(values.length, Optional.empty(), binaries)); + } + + private static TsBlock block(boolean runLengthEncodedTime, double... values) { + Column timeColumn = + runLengthEncodedTime + ? new RunLengthEncodedColumn(new TimeColumn(1, new long[] {0}), values.length) + : new TimeColumn(values.length, new long[values.length]); + return new TsBlock(timeColumn, doubleColumn(values)); + } + + private static void assertBlock(TsBlock actual, Double[]... expectedColumns) { + assertEquals(expectedColumns.length, actual.getValueColumnCount()); + assertEquals(expectedColumns[0].length, actual.getPositionCount()); + for (int columnIndex = 0; columnIndex < expectedColumns.length; columnIndex++) { + Column column = actual.getColumn(columnIndex); + Double[] expected = expectedColumns[columnIndex]; + assertEquals(expected.length, column.getPositionCount()); + for (int row = 0; row < expected.length; row++) { + if (expected[row] == null) { + assertTrue(column.isNull(row)); + } else { + assertFalse(column.isNull(row)); + assertEquals(expected[row], column.getDouble(row), 0.0); + } + } + } + } + + private static class Graph { + private final List leaves = new ArrayList<>(); + + private ColumnTransformer input(int index) { + return leaf(new IdentityColumnTransformer(DOUBLE, index)); + } + + private ColumnTransformer constant(double value) { + return leaf(new ConstantColumnTransformer(DOUBLE, doubleColumn(value))); + } + + private ColumnTransformer nullValue() { + return leaf(new NullColumnTransformer()); + } + + private ColumnTransformer leaf(LeafColumnTransformer transformer) { + leaves.add(transformer); + return transformer; + } + + private ColumnTransformer add(ColumnTransformer input, double value) { + return add(input, constant(value)); + } + + private ColumnTransformer add(ColumnTransformer left, ColumnTransformer right) { + return new ArithmeticAdditionColumnTransformer(DOUBLE, reference(left), reference(right)); + } + + private ColumnTransformer greaterThan(ColumnTransformer input, double value) { + return new CompareGreaterThanColumnTransformer( + BOOLEAN, reference(input), reference(constant(value))); + } + + private ColumnTransformer caseWhen( + ColumnTransformer when, ColumnTransformer then, ColumnTransformer otherwise) { + return new TableCaseWhenThenColumnTransformer( + DOUBLE, + Collections.singletonList(reference(when)), + Collections.singletonList(reference(then)), + reference(otherwise)); + } + } + + private static class CountingAdditionColumnTransformer + extends ArithmeticAdditionColumnTransformer { + private int fullEvaluationCount; + private int selectedEvaluationCount; + + private CountingAdditionColumnTransformer(ColumnTransformer left, ColumnTransformer right) { + super(DOUBLE, left, right); + } + + @Override + public void evaluate() { + fullEvaluationCount++; + super.evaluate(); + } + + @Override + protected Column evaluateSelection(boolean[] selection) { + selectedEvaluationCount++; + return super.evaluateSelection(selection); + } + } + + private static class CountingIdentityColumnTransformer extends IdentityColumnTransformer { + private int evaluationCount; + private int childTraversalCount; + + private CountingIdentityColumnTransformer() { + super(DOUBLE, 0); + } + + @Override + protected void evaluate() { + evaluationCount++; + super.evaluate(); + } + + @Override + protected void forEachChild(Consumer action) { + childTraversalCount++; + } + } + + private static class TestOperatorContext extends CommonOperatorContext { + private TestOperatorContext() { + super(0, new PlanNodeId("test"), "test"); + } + + @Override + public MemoryReservationManager getMemoryReservationContext() { + return null; + } + + @Override + public int getFragmentId() { + return 0; + } + + @Override + public int getPipelineId() { + return 0; + } + + @Override + public long ramBytesUsed() { + return 0; + } + } +} diff --git a/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/transformation/dag/column/ColumnTransformerTest.java b/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/transformation/dag/column/ColumnTransformerTest.java new file mode 100644 index 0000000000000..e797d5c124687 --- /dev/null +++ b/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/transformation/dag/column/ColumnTransformerTest.java @@ -0,0 +1,181 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.calc.transformation.dag.column; + +import org.apache.iotdb.calc.transformation.dag.column.binary.ArithmeticAdditionColumnTransformer; +import org.apache.iotdb.calc.transformation.dag.column.binary.LogicAndColumnTransformer; +import org.apache.iotdb.calc.transformation.dag.column.binary.LogicOrColumnTransformer; +import org.apache.iotdb.calc.transformation.dag.column.leaf.IdentityColumnTransformer; + +import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.block.column.ColumnBuilder; +import org.apache.tsfile.read.common.block.TsBlock; +import org.apache.tsfile.read.common.block.column.DoubleColumn; +import org.apache.tsfile.read.common.block.column.TimeColumn; +import org.junit.Test; + +import java.util.Arrays; +import java.util.Optional; + +import static org.apache.tsfile.read.common.type.BooleanType.BOOLEAN; +import static org.apache.tsfile.read.common.type.DoubleType.DOUBLE; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertSame; + +public class ColumnTransformerTest { + + @Test + public void testCollectionDeduplicatesByIdentity() { + ColumnTransformer first = new EqualIdentityColumnTransformer(0); + ColumnTransformer second = new EqualIdentityColumnTransformer(1); + ColumnTransformer root = new ArithmeticAdditionColumnTransformer(DOUBLE, first, second); + + ColumnTransformer[] collected = + ColumnTransformer.collectTransformers(Arrays.asList(root, first, root)); + + assertEquals(3, collected.length); + assertSame(root, collected[0]); + assertSame(first, collected[1]); + assertSame(second, collected[2]); + } + + @Test + public void testNewInputClearingOnlyClearsOwnCache() { + ColumnTransformer left = new IdentityColumnTransformer(DOUBLE, 0); + ColumnTransformer right = new IdentityColumnTransformer(DOUBLE, 1); + ColumnTransformer root = new ArithmeticAdditionColumnTransformer(DOUBLE, left, right); + left.addReferenceCount(); + right.addReferenceCount(); + root.addReferenceCount(); + left.initializeColumnCache(column(1)); + right.initializeColumnCache(column(2)); + root.initializeColumnCache(column(3)); + + root.clearCacheForNewInput(); + + assertEquals(0, root.getColumnCachePositionCount()); + assertEquals(1.0, left.getColumn().getDouble(0), 0.0); + assertEquals(2.0, right.getColumn().getDouble(0), 0.0); + } + + @Test + public void testSelectedResultDoesNotBecomeFullCache() { + IdentityColumnTransformer input = new IdentityColumnTransformer(DOUBLE, 0); + input.addReferenceCount(); + Column values = new DoubleColumn(2, Optional.empty(), new double[] {1, 2}); + input.initFromTsBlock(new TsBlock(new TimeColumn(2, new long[2]), values)); + + Column selected = input.getColumnWithSelection(new boolean[] {true, false}); + assertEquals(1.0, selected.getDouble(0), 0.0); + org.junit.Assert.assertTrue(selected.isNull(1)); + assertEquals(0, input.getColumnCachePositionCount()); + + input.tryEvaluate(); + // Borrowing a full cached column for a selected consumer must not consume its reference. + assertSame(values, input.getColumnWithSelection(new boolean[] {false, true})); + assertSame(values, input.getColumn()); + } + + @Test + public void testLegacySelectedEntryDoesNotReplacePendingFullColumn() { + IdentityColumnTransformer input = new IdentityColumnTransformer(DOUBLE, 0); + input.addReferenceCount(); + Column values = new DoubleColumn(2, Optional.empty(), new double[] {1, 2}); + input.initFromTsBlock(new TsBlock(new TimeColumn(2, new long[2]), values)); + input.tryEvaluate(); + input.evaluateWithSelection(new boolean[] {true, false}); + input.getColumn(); + input.tryEvaluate(); + assertSame(values, input.getColumn()); + } + + @Test + public void testSelectedLogicalOperatorsPreserveThreeValuedLogicWithCachedInputs() { + Boolean[] values = {null, false, true}; + ColumnBuilder left = BOOLEAN.createColumnBuilder(9); + ColumnBuilder right = BOOLEAN.createColumnBuilder(9); + Boolean[] expectedAnd = {null, false, null, false, false, false, null, false, true}; + Boolean[] expectedOr = {null, null, true, null, false, true, true, true, true}; + for (Boolean a : values) { + for (Boolean b : values) { + if (a == null) { + left.appendNull(); + } else { + left.writeBoolean(a); + } + if (b == null) { + right.appendNull(); + } else { + right.writeBoolean(b); + } + } + } + ColumnTransformer a = new IdentityColumnTransformer(BOOLEAN, 0); + ColumnTransformer b = new IdentityColumnTransformer(BOOLEAN, 1); + a.addReferenceCount(); + b.addReferenceCount(); + a.initializeColumnCache(left.build()); + b.initializeColumnCache(right.build()); + ColumnTransformer[] operators = { + new LogicAndColumnTransformer(BOOLEAN, a, b), new LogicOrColumnTransformer(BOOLEAN, a, b) + }; + for (int i = 0; i < operators.length; i++) { + Boolean[] expected = i == 0 ? expectedAnd : expectedOr; + for (boolean partial : new boolean[] {false, true}) { + boolean[] mask = new boolean[9]; + for (int j = 0; j < mask.length; j++) { + mask[j] = !partial || j % 2 == 0; + } + Column result = operators[i].getColumnWithSelection(mask); + assertEquals(9, result.getPositionCount()); + for (int j = 0; j < mask.length; j++) { + if (!mask[j] || expected[j] == null) { + org.junit.Assert.assertTrue(result.isNull(j)); + } else { + assertEquals(expected[j], result.getBoolean(j)); + } + } + } + } + // Selected consumers did not consume the two pending full-column references. + assertEquals(9, a.getColumn().getPositionCount()); + assertEquals(9, b.getColumn().getPositionCount()); + } + + private static Column column(double value) { + return new DoubleColumn(1, Optional.empty(), new double[] {value}); + } + + private static class EqualIdentityColumnTransformer extends IdentityColumnTransformer { + private EqualIdentityColumnTransformer(int index) { + super(DOUBLE, index); + } + + @Override + public boolean equals(Object other) { + return other instanceof EqualIdentityColumnTransformer; + } + + @Override + public int hashCode() { + return 1; + } + } +} diff --git a/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/transformation/dag/column/DiffColumnTransformerTest.java b/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/transformation/dag/column/DiffColumnTransformerTest.java new file mode 100644 index 0000000000000..26b72ad844169 --- /dev/null +++ b/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/transformation/dag/column/DiffColumnTransformerTest.java @@ -0,0 +1,156 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.calc.transformation.dag.column; + +import org.apache.iotdb.calc.transformation.dag.column.leaf.IdentityColumnTransformer; +import org.apache.iotdb.calc.transformation.dag.column.unary.scalar.DiffColumnTransformer; + +import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.block.column.ColumnBuilder; +import org.apache.tsfile.read.common.block.TsBlock; +import org.apache.tsfile.read.common.block.column.TimeColumn; +import org.junit.Test; + +import java.util.Arrays; + +import static org.apache.tsfile.read.common.type.BooleanType.BOOLEAN; +import static org.apache.tsfile.read.common.type.DoubleType.DOUBLE; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +public class DiffColumnTransformerTest { + @Test + public void testMasksAndConsumedFullCacheAdvanceStateOncePerInput() { + for (Boolean ignoreNull : new Boolean[] {true, false, null}) { + IdentityColumnTransformer input = new IdentityColumnTransformer(DOUBLE, 0); + IdentityColumnTransformer flags = new IdentityColumnTransformer(BOOLEAN, 1); + input.addReferenceCount(); + flags.addReferenceCount(); + DiffColumnTransformer diff = new DiffColumnTransformer(DOUBLE, input, flags); + diff.addReferenceCount(); + ColumnTransformer[] nodes = ColumnTransformer.collectTransformers(Arrays.asList(diff)); + TsBlock[] blocks = { + block(new Double[] {1.0, 3.0, null}, new Boolean[] {false, null, ignoreNull}), + block(new Double[] {10.0, null, 20.0, null}, new Boolean[] {false, true, false, null}), + block(new Double[] {30.0, 32.0}, new Boolean[] {true, false}), + block(new Double[] {40.0, 41.0}, new Boolean[] {false, true}) + }; + Double[][] expected = { + {null, 2.0, null}, + {Boolean.TRUE.equals(ignoreNull) ? 7.0 : null, null, 10.0, null}, + {null, 2.0}, + {8.0, 1.0} + }; + for (int i = 0; i < blocks.length; i++) { + for (ColumnTransformer node : nodes) { + node.clearCacheForNewInput(); + } + input.initFromTsBlock(blocks[i]); + flags.initFromTsBlock(blocks[i]); + boolean[] mask = new boolean[blocks[i].getPositionCount()]; + // Even an all-false mask must advance DIFF across the complete input exactly once. + assertValues(diff.getColumnWithSelection(mask), expected[i]); + Arrays.fill(mask, true); + assertValues(diff.getColumnWithSelection(mask), expected[i]); + diff.tryEvaluate(); + assertValues(diff.getColumn(), expected[i]); + diff.tryEvaluate(); + assertValues(diff.getColumn(), expected[i]); + // Clearing ordinary caches within the same input must not advance state again. + diff.clearCache(); + assertValues(diff.getColumnWithSelection(mask), expected[i]); + diff.tryEvaluate(); + assertValues(diff.getColumn(), expected[i]); + } + diff.close(); + } + } + + @Test + public void testPartialSelectionDoesNotMaskInputs() { + IdentityColumnTransformer input = new IdentityColumnTransformer(DOUBLE, 0); + IdentityColumnTransformer flags = new IdentityColumnTransformer(BOOLEAN, 1); + input.addReferenceCount(); + flags.addReferenceCount(); + TsBlock block = block(new Double[] {1.0, 3.0, 6.0}, new Boolean[] {true, false, true}); + input.initFromTsBlock(block); + flags.initFromTsBlock(block); + DiffColumnTransformer diff = new DiffColumnTransformer(DOUBLE, input, flags); + diff.addReferenceCount(); + assertValues( + diff.getColumnWithSelection(new boolean[] {false, true, false}), + new Double[] {null, 2.0, 3.0}); + diff.tryEvaluate(); + assertValues(diff.getColumn(), new Double[] {null, 2.0, 3.0}); + diff.close(); + } + + @Test + public void testConsumedFullCacheDoesNotAdvanceStateAgain() { + IdentityColumnTransformer input = new IdentityColumnTransformer(DOUBLE, 0); + IdentityColumnTransformer flags = new IdentityColumnTransformer(BOOLEAN, 1); + input.addReferenceCount(); + flags.addReferenceCount(); + TsBlock block = block(new Double[] {1.0, 3.0, 6.0}, new Boolean[] {true, false, true}); + input.initFromTsBlock(block); + flags.initFromTsBlock(block); + DiffColumnTransformer diff = new DiffColumnTransformer(DOUBLE, input, flags); + diff.addReferenceCount(); + diff.tryEvaluate(); + assertValues(diff.getColumn(), new Double[] {null, 2.0, 3.0}); + diff.tryEvaluate(); + assertValues(diff.getColumn(), new Double[] {null, 2.0, 3.0}); + diff.close(); + } + + private static TsBlock block(Double[] values, Boolean[] flags) { + ColumnBuilder valueBuilder = DOUBLE.createColumnBuilder(values.length); + ColumnBuilder flagBuilder = BOOLEAN.createColumnBuilder(values.length); + for (int i = 0; i < values.length; i++) { + if (values[i] == null) { + valueBuilder.appendNull(); + } else { + valueBuilder.writeDouble(values[i]); + } + if (flags[i] == null) { + flagBuilder.appendNull(); + } else { + flagBuilder.writeBoolean(flags[i]); + } + } + return new TsBlock( + new TimeColumn(values.length, new long[values.length]), + valueBuilder.build(), + flagBuilder.build()); + } + + private static void assertValues(Column column, Double[] expected) { + assertEquals(expected.length, column.getPositionCount()); + for (int i = 0; i < expected.length; i++) { + if (expected[i] == null) { + assertTrue(column.isNull(i)); + } else { + assertFalse(column.isNull(i)); + assertEquals(expected[i], column.getDouble(i), 0.0); + } + } + } +} diff --git a/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/transformation/dag/column/DiffFunctionColumnTransformerTest.java b/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/transformation/dag/column/DiffFunctionColumnTransformerTest.java new file mode 100644 index 0000000000000..0210d4ea16a2d --- /dev/null +++ b/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/transformation/dag/column/DiffFunctionColumnTransformerTest.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.calc.transformation.dag.column; + +import org.apache.iotdb.calc.transformation.dag.column.leaf.IdentityColumnTransformer; +import org.apache.iotdb.calc.transformation.dag.column.unary.scalar.DiffFunctionColumnTransformer; + +import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.read.common.block.TsBlock; +import org.apache.tsfile.read.common.block.column.DoubleColumn; +import org.apache.tsfile.read.common.block.column.TimeColumn; +import org.junit.Test; + +import java.util.Arrays; +import java.util.Optional; + +import static org.apache.tsfile.read.common.type.DoubleType.DOUBLE; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class DiffFunctionColumnTransformerTest { + @Test + public void testMasksAndConsumedFullCacheAdvanceStateOncePerInput() { + for (boolean ignoreNull : new boolean[] {true, false}) { + IdentityColumnTransformer input = new IdentityColumnTransformer(DOUBLE, 0); + input.addReferenceCount(); + DiffFunctionColumnTransformer diff = + new DiffFunctionColumnTransformer(DOUBLE, input, ignoreNull); + diff.addReferenceCount(); + ColumnTransformer[] nodes = ColumnTransformer.collectTransformers(Arrays.asList(diff)); + TsBlock[] blocks = { + block(new double[] {1, 3, 0}, new boolean[] {false, false, true}), + block(new double[] {10, 11}, null) + }; + Double[][] expected = {{null, 2.0, null}, {ignoreNull ? 7.0 : null, 1.0}}; + for (int i = 0; i < blocks.length; i++) { + for (ColumnTransformer node : nodes) { + node.clearCacheForNewInput(); + } + input.initFromTsBlock(blocks[i]); + boolean[] mask = new boolean[blocks[i].getPositionCount()]; + // Even an all-false mask must advance DIFF across all input rows exactly once. + assertValues(diff.getColumnWithSelection(mask), expected[i]); + Arrays.fill(mask, true); + assertValues(diff.getColumnWithSelection(mask), expected[i]); + diff.tryEvaluate(); + assertValues(diff.getColumn(), expected[i]); + // The ordinary cache has been consumed; re-requesting it must not advance state again. + diff.tryEvaluate(); + assertValues(diff.getColumn(), expected[i]); + assertValues(diff.getColumnWithSelection(mask), expected[i]); + // Ordinary cache clearing does not mark the start of a new input. + diff.clearCache(); + diff.tryEvaluate(); + assertValues(diff.getColumn(), expected[i]); + } + } + } + + private static TsBlock block(double[] values, boolean[] nulls) { + return new TsBlock( + new TimeColumn(values.length, new long[values.length]), + new DoubleColumn(values.length, Optional.ofNullable(nulls), values)); + } + + private static void assertValues(Column column, Double[] expected) { + assertEquals(expected.length, column.getPositionCount()); + for (int i = 0; i < expected.length; i++) { + if (expected[i] == null) { + assertTrue(column.isNull(i)); + } else { + assertEquals(expected[i], column.getDouble(i), 0.0); + } + } + } +} diff --git a/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/transformation/dag/column/MappableUDFColumnTransformerTest.java b/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/transformation/dag/column/MappableUDFColumnTransformerTest.java new file mode 100644 index 0000000000000..6d0d3c208589b --- /dev/null +++ b/iotdb-core/calc-commons/src/test/java/org/apache/iotdb/calc/transformation/dag/column/MappableUDFColumnTransformerTest.java @@ -0,0 +1,331 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.calc.transformation.dag.column; + +import org.apache.iotdb.calc.transformation.dag.column.leaf.IdentityColumnTransformer; +import org.apache.iotdb.calc.transformation.dag.column.leaf.TimeColumnTransformer; +import org.apache.iotdb.calc.transformation.dag.column.multi.MappableUDFColumnTransformer; +import org.apache.iotdb.calc.transformation.dag.column.unary.scalar.DiffFunctionColumnTransformer; +import org.apache.iotdb.calc.transformation.dag.udf.UDTFExecutor; +import org.apache.iotdb.commons.udf.builtin.String.UDTFLower; +import org.apache.iotdb.commons.udf.builtin.String.UDTFUpper; +import org.apache.iotdb.commons.udf.builtin.UDTFAbs; +import org.apache.iotdb.udf.api.UDTF; +import org.apache.iotdb.udf.api.customizer.parameter.UDFParameterValidator; +import org.apache.iotdb.udf.api.customizer.parameter.UDFParameters; + +import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.block.column.ColumnBuilder; +import org.apache.tsfile.common.conf.TSFileConfig; +import org.apache.tsfile.read.common.block.TsBlock; +import org.apache.tsfile.read.common.block.column.DictionaryColumn; +import org.apache.tsfile.read.common.block.column.RunLengthEncodedColumn; +import org.apache.tsfile.read.common.block.column.TimeColumn; +import org.apache.tsfile.read.common.type.Type; +import org.apache.tsfile.utils.Binary; +import org.junit.Test; + +import java.time.ZoneId; +import java.util.Arrays; +import java.util.Collections; + +import static org.apache.tsfile.read.common.type.BinaryType.TEXT; +import static org.apache.tsfile.read.common.type.DoubleType.DOUBLE; +import static org.apache.tsfile.read.common.type.LongType.INT64; +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; + +public class MappableUDFColumnTransformerTest { + + @Test + public void testAbsWithSelectedRowsAndNulls() throws Exception { + Column values = doubles(-10.0, -20.0, null, -40.0); + for (boolean cached : new boolean[] {false, true}) { + assertDoubles( + transform( + new UDTFAbs(), DOUBLE, values, cached, new boolean[] {false, true, false, true}), + null, + 20.0, + null, + 40.0); + assertDoubles( + transform( + new UDTFAbs(), DOUBLE, values, cached, new boolean[] {false, true, false, false}), + null, + 20.0, + null, + null); + assertDoubles( + transform(new UDTFAbs(), DOUBLE, values, cached, new boolean[] {false, true, true, true}), + null, + 20.0, + null, + 40.0); + } + } + + @Test + public void testAbsWithDictionaryOffsetAndRunLengthEncodedInputs() throws Exception { + Column dictionary = DictionaryColumn.create(4, doubles(-10.0, -20.0), new int[] {1, 0, 1, 0}); + assertDoubles( + transform(new UDTFAbs(), DOUBLE, dictionary, true, new boolean[] {false, true, true, true}), + null, + 10.0, + 20.0, + 10.0); + + Column offset = doubles(-100.0, -10.0, -20.0, -30.0, -40.0).getRegion(1, 4); + assertDoubles( + transform(new UDTFAbs(), DOUBLE, offset, true, new boolean[] {false, true, true, false}), + null, + 20.0, + 30.0, + null); + + Column repeated = new RunLengthEncodedColumn(doubles(-7.0), 4); + assertDoubles( + transform(new UDTFAbs(), DOUBLE, repeated, true, new boolean[] {false, true, true, false}), + null, + 7.0, + 7.0, + null); + assertDoubles( + transform( + new UDTFAbs(), + DOUBLE, + doubles(null, null, null), + true, + new boolean[] {true, true, true}), + null, + null, + null); + } + + @Test + public void testStringBuiltinsWithSelectedRows() throws Exception { + ColumnBuilder builder = TEXT.createColumnBuilder(4); + builder.writeBinary(new Binary("FIRST", TSFileConfig.STRING_CHARSET)); + builder.writeBinary(new Binary("MiXeD", TSFileConfig.STRING_CHARSET)); + builder.appendNull(); + builder.writeBinary(new Binary("LAST", TSFileConfig.STRING_CHARSET)); + Column values = builder.build(); + for (boolean cached : new boolean[] {false, true}) { + assertStrings( + transform(new UDTFLower(), TEXT, values, cached, new boolean[] {false, true, true, true}), + null, + "mixed", + null, + "last"); + assertStrings( + transform( + new UDTFUpper(), TEXT, values, cached, new boolean[] {false, true, false, false}), + null, + "MIXED", + null, + null); + } + } + + @Test + public void testAllAndNoSelectedRows() throws Exception { + RecordingAbs udf = new RecordingAbs(); + Column values = doubles(-10.0, -20.0, -30.0); + assertDoubles( + transform(udf, DOUBLE, values, true, new boolean[] {false, false, false}), + null, + null, + null); + assertEquals(0, udf.invocations); + + assertDoubles( + transform(udf, DOUBLE, values, true, new boolean[] {true, true, true}), 10.0, 20.0, 30.0); + assertEquals(1, udf.invocations); + assertArrayEquals(new long[] {10, 20, 30}, udf.times); + } + + @Test + public void testTimeColumnOnlyContainsSelectedPositions() throws Exception { + RecordingAbs udf = new RecordingAbs(); + assertDoubles( + transform( + udf, + DOUBLE, + doubles(-10.0, -20.0, -30.0, -40.0), + true, + new boolean[] {false, true, false, true}), + null, + 20.0, + null, + 40.0); + assertArrayEquals(new long[] {20, 40}, udf.times); + + assertDoubles( + transform( + udf, DOUBLE, doubles(-10.0, -20.0, -30.0), false, new boolean[] {false, false, true}), + null, + null, + 30.0); + assertArrayEquals(new long[] {30}, udf.times); + } + + @Test + public void testNoSelectedRowsStillAdvanceDiffInput() throws Exception { + IdentityColumnTransformer input = new IdentityColumnTransformer(DOUBLE, 0); + input.addReferenceCount(); + TimeColumnTransformer time = new TimeColumnTransformer(INT64); + time.addReferenceCount(); + DiffFunctionColumnTransformer diff = new DiffFunctionColumnTransformer(DOUBLE, input, true); + diff.addReferenceCount(); + RecordingAbs udf = new RecordingAbs(); + MappableUDFColumnTransformer transformer = + new MappableUDFColumnTransformer( + DOUBLE, new ColumnTransformer[] {diff, time}, new TestExecutor(udf, DOUBLE)); + try { + TsBlock first = block(doubles(-10.0, -20.0, -30.0)); + input.initFromTsBlock(first); + time.initFromTsBlock(first); + assertDoubles( + transformer.getColumnWithSelection(new boolean[] {false, false, false}), + null, + null, + null); + assertEquals(0, udf.invocations); + + for (ColumnTransformer node : + ColumnTransformer.collectTransformers(Collections.singletonList(transformer))) { + node.clearCacheForNewInput(); + } + TsBlock second = block(doubles(-50.0, -70.0)); + input.initFromTsBlock(second); + time.initFromTsBlock(second); + assertDoubles(transformer.getColumnWithSelection(new boolean[] {true, true}), 20.0, 20.0); + assertEquals(1, udf.invocations); + } finally { + transformer.close(); + } + } + + private static Column transform( + UDTF udf, Type type, Column values, boolean cached, boolean[] selection) throws Exception { + IdentityColumnTransformer input = new IdentityColumnTransformer(type, 0); + input.addReferenceCount(); + TimeColumnTransformer time = new TimeColumnTransformer(INT64); + time.addReferenceCount(); + TsBlock block = block(values); + input.initFromTsBlock(block); + time.initFromTsBlock(block); + if (cached) { + input.tryEvaluate(); + time.tryEvaluate(); + } + MappableUDFColumnTransformer transformer = + new MappableUDFColumnTransformer( + type, new ColumnTransformer[] {input, time}, new TestExecutor(udf, type)); + try { + Column result = transformer.getColumnWithSelection(selection); + if (cached) { + // Selected evaluation must not consume either pending full-column reference. + assertSame(values, input.getColumn()); + assertSame(block.getTimeColumn(), time.getColumn()); + } + return result; + } finally { + transformer.close(); + } + } + + private static TsBlock block(Column values) { + long[] times = new long[values.getPositionCount()]; + for (int i = 0; i < times.length; i++) { + times[i] = (i + 1) * 10L; + } + return new TsBlock(new TimeColumn(times.length, times), values); + } + + private static Column doubles(Double... values) { + ColumnBuilder builder = DOUBLE.createColumnBuilder(values.length); + for (Double value : values) { + if (value == null) { + builder.appendNull(); + } else { + builder.writeDouble(value); + } + } + return builder.build(); + } + + private static void assertDoubles(Column result, Double... expected) { + assertEquals(expected.length, result.getPositionCount()); + for (int i = 0; i < expected.length; i++) { + if (expected[i] == null) { + assertTrue(result.isNull(i)); + } else { + assertFalse(result.isNull(i)); + assertEquals(expected[i], result.getDouble(i), 0.0); + } + } + } + + private static void assertStrings(Column result, String... expected) { + assertEquals(expected.length, result.getPositionCount()); + for (int i = 0; i < expected.length; i++) { + if (expected[i] == null) { + assertTrue(result.isNull(i)); + } else { + assertFalse(result.isNull(i)); + assertEquals(expected[i], result.getBinary(i).getStringValue(TSFileConfig.STRING_CHARSET)); + } + } + } + + private static class TestExecutor extends UDTFExecutor { + private TestExecutor(UDTF udf, Type type) throws Exception { + super("selection_test", ZoneId.of("UTC")); + this.udtf = udf; + UDFParameters parameters = + new UDFParameters( + Collections.singletonList("input"), + Collections.singletonList( + type == DOUBLE + ? org.apache.iotdb.udf.api.type.Type.DOUBLE + : org.apache.iotdb.udf.api.type.Type.TEXT), + Collections.emptyMap()); + udf.validate(new UDFParameterValidator(parameters)); + udf.beforeStart(parameters, configurations); + configurations.check(); + } + } + + private static class RecordingAbs extends UDTFAbs { + private int invocations; + private long[] times; + + @Override + public void transform(Column[] columns, ColumnBuilder builder) throws Exception { + invocations++; + Column time = columns[columns.length - 1]; + times = Arrays.copyOf(time.getLongs(), time.getPositionCount()); + super.transform(columns, builder); + } + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DeviceAttributeUpdater.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DeviceAttributeUpdater.java index 9120444ba0d23..60486568f00b3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DeviceAttributeUpdater.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DeviceAttributeUpdater.java @@ -51,6 +51,7 @@ public class DeviceAttributeUpdater extends DeviceUpdater { private final List commonTransformerList; private final List projectLeafColumnTransformerList; private final List projectOutputTransformerList; + private final ColumnTransformer[] projectCacheTransformers; private final TriConsumer attributeUpdater; private final TsBlockBuilder filterTsBlockBuilder; private final List attributePointers = new ArrayList<>(); @@ -75,6 +76,7 @@ public DeviceAttributeUpdater( super( filterLeafColumnTransformerList, filterOutputTransformer, + commonTransformerList, columnSchemaList, attributeProvider, database, @@ -82,6 +84,8 @@ public DeviceAttributeUpdater( this.commonTransformerList = commonTransformerList; this.projectLeafColumnTransformerList = projectLeafColumnTransformerList; this.projectOutputTransformerList = projectOutputTransformerList; + this.projectCacheTransformers = + ColumnTransformer.collectTransformers(projectOutputTransformerList); this.attributeUpdater = attributeUpdater; this.filterTsBlockBuilder = new TsBlockBuilder(filterOutputDataTypes); this.attributeNames = attributeNames; @@ -97,6 +101,9 @@ public void handleDeviceNode(final IDeviceMNode node) throws Metadata protected void update() throws MetadataException { final TsBlock block = getFilterTsBlock(); + for (ColumnTransformer transformer : projectCacheTransformers) { + transformer.clearCacheForNewInput(); + } projectLeafColumnTransformerList.forEach( leafColumnTransformer -> leafColumnTransformer.initFromTsBlock(block)); @@ -145,9 +152,15 @@ private TsBlock getFilterTsBlock() { final List filterResultColumns = new ArrayList<>(Arrays.asList(curBlock.getValueColumns())); - // Get result of calculated common sub expressions - commonTransformerList.forEach( - columnTransformer -> filterResultColumns.add(columnTransformer.getColumn())); + if (!commonTransformerList.isEmpty()) { + boolean[] selection = new boolean[deviceSchemaBatch.size()]; + for (int i = 0; i < selection.length; i++) { + selection[i] = !curFilterColumn.isNull(i) && curFilterColumn.getBoolean(i); + } + for (ColumnTransformer transformer : commonTransformerList) { + filterResultColumns.add(transformer.getColumnWithSelection(selection)); + } + } final ColumnBuilder[] columnBuilders = filterTsBlockBuilder.getValueColumnBuilders(); final int rowCount = diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DeviceBlackListConstructor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DeviceBlackListConstructor.java index b8f315b3a0c7d..324baca9be7d1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DeviceBlackListConstructor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DeviceBlackListConstructor.java @@ -31,6 +31,7 @@ import org.apache.tsfile.utils.Binary; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.function.BiFunction; @@ -54,6 +55,7 @@ public DeviceBlackListConstructor( super( filterLeafColumnTransformerList, filterOutputTransformer, + Collections.emptyList(), columnSchemaList, attributeProvider, database, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DevicePredicateFilter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DevicePredicateFilter.java index 589f119c3ae14..a9db01b879a49 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DevicePredicateFilter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DevicePredicateFilter.java @@ -25,6 +25,7 @@ import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema; import org.apache.iotdb.db.schemaengine.schemaregion.read.resp.info.IDeviceSchemaInfo; +import java.util.Collections; import java.util.Iterator; import java.util.List; import java.util.NoSuchElementException; @@ -44,6 +45,7 @@ public DevicePredicateFilter( super( filterLeafColumnTransformerList, filterOutputTransformer, + Collections.emptyList(), columnSchemaList, database, table); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DevicePredicateHandler.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DevicePredicateHandler.java index db7d0a2b374bf..b3cf75f120cc6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DevicePredicateHandler.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DevicePredicateHandler.java @@ -35,6 +35,7 @@ import org.apache.tsfile.read.common.block.TsBlockBuilder; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Objects; import java.util.stream.Collectors; @@ -45,6 +46,7 @@ public abstract class DevicePredicateHandler implements AutoCloseable { private final List filterLeafColumnTransformerList; protected final ColumnTransformer filterOutputTransformer; + private final ColumnTransformer[] filterCacheTransformers; private final List inputDataTypes; private final List columnSchemaList; private final int tagIndex; @@ -59,14 +61,36 @@ public abstract class DevicePredicateHandler implements AutoCloseable { protected TsBlock curBlock; protected Column curFilterColumn; + // Retain the original constructor descriptor for precompiled subclasses without common roots. protected DevicePredicateHandler( final List filterLeafColumnTransformerList, final ColumnTransformer filterOutputTransformer, final List columnSchemaList, final String database, final TsTable table) { + this( + filterLeafColumnTransformerList, + filterOutputTransformer, + Collections.emptyList(), + columnSchemaList, + database, + table); + } + + protected DevicePredicateHandler( + final List filterLeafColumnTransformerList, + final ColumnTransformer filterOutputTransformer, + final List commonTransformerList, + final List columnSchemaList, + final String database, + final TsTable table) { this.filterLeafColumnTransformerList = filterLeafColumnTransformerList; this.filterOutputTransformer = filterOutputTransformer; + List filterRoots = new ArrayList<>(commonTransformerList); + if (filterOutputTransformer != null) { + filterRoots.add(filterOutputTransformer); + } + this.filterCacheTransformers = ColumnTransformer.collectTransformers(filterRoots); this.columnSchemaList = columnSchemaList; this.inputDataTypes = columnSchemaList.stream() @@ -106,6 +130,10 @@ public void prepareBatchResult() { return; } + // A CASE can leave shared expressions cached after the previous batch was consumed. + for (ColumnTransformer transformer : filterCacheTransformers) { + transformer.clearCacheForNewInput(); + } // feed Filter ColumnTransformer, including TimeStampColumnTransformer and constant filterLeafColumnTransformerList.forEach( leafColumnTransformer -> leafColumnTransformer.initFromTsBlock(curBlock)); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DeviceUpdater.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DeviceUpdater.java index 2679f501dd060..c6690634e91cc 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DeviceUpdater.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DeviceUpdater.java @@ -31,6 +31,7 @@ import org.apache.tsfile.utils.Binary; +import java.util.Collections; import java.util.List; import java.util.function.BiFunction; @@ -38,6 +39,7 @@ public abstract class DeviceUpdater extends DevicePredicateHandler { private final BiFunction attributeProvider; + // Retain the original constructor descriptor for precompiled subclasses without common roots. protected DeviceUpdater( final List filterLeafColumnTransformerList, final ColumnTransformer filterOutputTransformer, @@ -45,9 +47,28 @@ protected DeviceUpdater( final BiFunction attributeProvider, final String database, final TsTable table) { + this( + filterLeafColumnTransformerList, + filterOutputTransformer, + Collections.emptyList(), + columnSchemaList, + attributeProvider, + database, + table); + } + + protected DeviceUpdater( + final List filterLeafColumnTransformerList, + final ColumnTransformer filterOutputTransformer, + final List commonTransformerList, + final List columnSchemaList, + final BiFunction attributeProvider, + final String database, + final TsTable table) { super( filterLeafColumnTransformerList, filterOutputTransformer, + commonTransformerList, columnSchemaList, database, table); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DevicePredicateHandlerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DevicePredicateHandlerTest.java new file mode 100644 index 0000000000000..149062fa53e48 --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DevicePredicateHandlerTest.java @@ -0,0 +1,324 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.execution.operator.schema.source; + +import org.apache.iotdb.calc.transformation.dag.column.ColumnTransformer; +import org.apache.iotdb.calc.transformation.dag.column.TableCaseWhenThenColumnTransformer; +import org.apache.iotdb.calc.transformation.dag.column.binary.CompareEqualToColumnTransformer; +import org.apache.iotdb.calc.transformation.dag.column.leaf.ConstantColumnTransformer; +import org.apache.iotdb.calc.transformation.dag.column.leaf.IdentityColumnTransformer; +import org.apache.iotdb.calc.transformation.dag.column.leaf.LeafColumnTransformer; +import org.apache.iotdb.calc.transformation.dag.column.unary.scalar.UpperColumnTransformer; +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.schema.node.role.IDeviceMNode; +import org.apache.iotdb.commons.schema.table.TsTable; +import org.apache.iotdb.commons.schema.table.column.TagColumnSchema; +import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema; +import org.apache.iotdb.db.schemaengine.rescon.MemSchemaRegionStatistics; +import org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.mem.mnode.IMemMNode; +import org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.mem.mnode.info.TableDeviceInfo; +import org.apache.iotdb.db.schemaengine.schemaregion.read.resp.info.impl.ShowDevicesResult; + +import org.apache.tsfile.common.conf.TSFileConfig; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.block.column.BinaryColumn; +import org.apache.tsfile.read.common.block.column.BooleanColumn; +import org.apache.tsfile.utils.Binary; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Optional; + +import static org.apache.tsfile.read.common.type.BooleanType.BOOLEAN; +import static org.apache.tsfile.read.common.type.StringType.STRING; +import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class DevicePredicateHandlerTest { + + private static final List COLUMNS = + Collections.singletonList(new TagColumnSchema("tag", TSDataType.STRING)); + + @Test + public void testPredicateFilterDoesNotReuseFullBatches() throws Exception { + Graph graph = new Graph(); + ColumnTransformer predicate = graph.matchesCase(graph.input(0)); + try (DevicePredicateFilter filter = predicateFilter(graph, predicate)) { + int batchSize = DevicePredicateHandler.DEFAULT_MAX_TS_BLOCK_LINE_NUMBER; + for (String value : new String[] {"match", "other"}) { + for (int i = 0; i < batchSize; i++) { + filter.addBatch(device(value)); + } + // addBatch evaluates full batches automatically; exhausting the iterator resets the batch. + List result = consume(filter); + assertEquals(batchSize, result.size()); + for (String actual : result) { + assertEquals(value, actual); + } + } + } + } + + @Test + public void testPredicateFilterDoesNotReuseDifferentSizedBatches() throws Exception { + Graph graph = new Graph(); + ColumnTransformer predicate = graph.matchesCase(graph.input(0)); + try (DevicePredicateFilter filter = predicateFilter(graph, predicate)) { + assertEquals(Collections.singletonList("match"), filterBatch(filter, "match")); + assertEquals(Arrays.asList("other", "match"), filterBatch(filter, "other", "match")); + assertEquals(Collections.emptyList(), filterBatch(filter, "skip")); + } + } + + @Test + public void testBlackListConstructorDoesNotReusePredicateAcrossBatches() throws Exception { + Graph graph = new Graph(); + ColumnTransformer predicate = graph.matchesCase(graph.input(0)); + List> nodes = + Arrays.asList(node("match", 0), node("other", 1), node("match", 2)); + try (DeviceBlackListConstructor constructor = + new DeviceBlackListConstructor( + graph.leaves, + reference(predicate), + "table", + COLUMNS, + (pointer, name) -> null, + mock(MemSchemaRegionStatistics.class), + "test", + new TsTable("table"))) { + constructor.handleDeviceNode(nodes.get(0)); + constructor.prepareBatchResult(); + constructor.update(); + constructor.handleDeviceNode(nodes.get(1)); + constructor.handleDeviceNode(nodes.get(2)); + constructor.prepareBatchResult(); + constructor.update(); + assertEquals(3, constructor.getPreDeletedNum()); + for (IDeviceMNode node : nodes) { + verify(node).preDeactivateSelfOrTemplate(); + } + } + } + + @Test + public void testAttributeUpdaterDoesNotReuseFilterAcrossBatches() throws Exception { + Graph filter = new Graph(); + ColumnTransformer predicate = filter.matchesCase(filter.input(0)); + Graph projection = new Graph(); + List> updates = new ArrayList<>(); + try (DeviceAttributeUpdater updater = + attributeUpdater( + filter, predicate, Collections.emptyList(), projection, updates, projection.input(0))) { + updateBatch(updater, "match"); + updateBatch(updater, "other", "match"); + assertEquals( + Arrays.asList( + Arrays.asList("match", "match"), + Arrays.asList("other", "other"), + Arrays.asList("match", "match")), + updates); + } + } + + @Test + public void testAttributeUpdaterDoesNotReuseProjectionAcrossBatches() throws Exception { + Graph projection = new Graph(); + ColumnTransformer input = projection.input(0); + ColumnTransformer caseWhen = projection.caseResult(input); + List> updates = new ArrayList<>(); + try (DeviceAttributeUpdater updater = + attributeUpdater( + null, null, Collections.emptyList(), projection, updates, caseWhen, input)) { + updateBatch(updater, "match"); + updateBatch(updater, "other", "match"); + assertEquals( + Arrays.asList( + Arrays.asList("match", "match", "match"), + Arrays.asList("other", "other", "other"), + Arrays.asList("match", "match", "match")), + updates); + } + } + + @Test + public void testAttributeUpdaterRecomputesCommonExpressionClearedByCase() throws Exception { + Graph filter = new Graph(); + ColumnTransformer shared = new UpperColumnTransformer(STRING, reference(filter.input(0))); + ColumnTransformer predicate = + filter.caseWhen( + filter.equal(shared, filter.constant("MATCH")), + filter.constant(true), + filter.constant(false)); + Graph projection = new Graph(); + List> updates = new ArrayList<>(); + try (DeviceAttributeUpdater updater = + attributeUpdater( + filter, + predicate, + Collections.singletonList(shared), + projection, + updates, + projection.input(1))) { + updateBatch(updater, "match"); + updateBatch(updater, "other", "match"); + assertEquals( + Arrays.asList(Arrays.asList("match", "MATCH"), Arrays.asList("match", "MATCH")), updates); + } + } + + private static DevicePredicateFilter predicateFilter(Graph graph, ColumnTransformer predicate) { + return new DevicePredicateFilter( + graph.leaves, reference(predicate), COLUMNS, "test", new TsTable("table")); + } + + private static List filterBatch(DevicePredicateFilter filter, String... values) { + for (String value : values) { + filter.addBatch(device(value)); + } + filter.prepareBatchResult(); + return consume(filter); + } + + private static List consume(DevicePredicateFilter filter) { + List result = new ArrayList<>(); + while (filter.hasNext()) { + result.add(filter.next().getRawNodes()[3]); + } + return result; + } + + private static DeviceAttributeUpdater attributeUpdater( + Graph filter, + ColumnTransformer predicate, + List common, + Graph projection, + List> updates, + ColumnTransformer... outputs) { + List projectOutputs = new ArrayList<>(); + for (ColumnTransformer output : outputs) { + projectOutputs.add(reference(output)); + } + common.forEach(ColumnTransformer::addReferenceCount); + return new DeviceAttributeUpdater( + Collections.nCopies(1 + common.size(), TSDataType.STRING), + filter == null ? Collections.emptyList() : filter.leaves, + predicate == null ? null : reference(predicate), + common, + COLUMNS, + projection.leaves, + projectOutputs, + (pointer, name) -> null, + (deviceId, pointer, values) -> { + List row = new ArrayList<>(); + row.add(deviceId[0]); + for (Object value : values) { + row.add(((Binary) value).getStringValue(TSFileConfig.STRING_CHARSET)); + } + updates.add(row); + }, + Collections.nCopies(outputs.length, "attribute"), + "test", + new TsTable("table")); + } + + private static void updateBatch(DeviceAttributeUpdater updater, String... values) + throws Exception { + for (int i = 0; i < values.length; i++) { + updater.handleDeviceNode(node(values[i], i)); + } + updater.prepareBatchResult(); + updater.update(); + } + + private static ShowDevicesResult device(String value) { + return new ShowDevicesResult(null, true, 0, new String[] {"root", "test", "table", value}); + } + + @SuppressWarnings("unchecked") + private static IDeviceMNode node(String value, int pointer) { + IDeviceMNode node = mock(IDeviceMNode.class); + TableDeviceInfo info = new TableDeviceInfo<>(); + info.setAttributePointer(pointer); + when(node.getDeviceInfo()).thenReturn(info); + when(node.getPartialPath()).thenReturn(new PartialPath(device(value).getRawNodes())); + when(node.isAlignedNullable()).thenReturn(true); + return node; + } + + private static ColumnTransformer reference(ColumnTransformer transformer) { + transformer.addReferenceCount(); + return transformer; + } + + private static class Graph { + private final List leaves = new ArrayList<>(); + + private ColumnTransformer input(int index) { + return leaf(new IdentityColumnTransformer(STRING, index)); + } + + private ColumnTransformer constant(String value) { + return leaf( + new ConstantColumnTransformer( + STRING, + new BinaryColumn( + 1, + Optional.empty(), + new Binary[] {new Binary(value, TSFileConfig.STRING_CHARSET)}))); + } + + private ColumnTransformer constant(boolean value) { + return leaf( + new ConstantColumnTransformer( + BOOLEAN, new BooleanColumn(1, Optional.empty(), new boolean[] {value}))); + } + + private ColumnTransformer leaf(LeafColumnTransformer transformer) { + leaves.add(transformer); + return transformer; + } + + private ColumnTransformer equal(ColumnTransformer left, ColumnTransformer right) { + return new CompareEqualToColumnTransformer(BOOLEAN, reference(left), reference(right)); + } + + private ColumnTransformer caseWhen( + ColumnTransformer when, ColumnTransformer then, ColumnTransformer otherwise) { + return new TableCaseWhenThenColumnTransformer( + then.getType(), + Collections.singletonList(reference(when)), + Collections.singletonList(reference(then)), + reference(otherwise)); + } + + private ColumnTransformer caseResult(ColumnTransformer input) { + return caseWhen(equal(input, constant("match")), constant("match"), constant("other")); + } + + private ColumnTransformer matchesCase(ColumnTransformer input) { + return equal(caseResult(input), input); + } + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/UserDefineScalarFunctionColumnTransformerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/UserDefineScalarFunctionColumnTransformerTest.java new file mode 100644 index 0000000000000..fda1cf8232838 --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/UserDefineScalarFunctionColumnTransformerTest.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.transformation.dag.column; + +import org.apache.iotdb.calc.execution.relational.ColumnTransformerBuilder; +import org.apache.iotdb.calc.transformation.dag.column.leaf.IdentityColumnTransformer; +import org.apache.iotdb.calc.transformation.dag.column.udf.UserDefineScalarFunctionTransformer; +import org.apache.iotdb.udf.api.IoTDBLocal; +import org.apache.iotdb.udf.api.relational.ScalarFunction; +import org.apache.iotdb.udf.api.relational.access.Record; + +import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.read.common.block.TsBlock; +import org.apache.tsfile.read.common.block.column.DoubleColumn; +import org.apache.tsfile.read.common.block.column.TimeColumn; +import org.junit.Test; +import org.mockito.Mockito; + +import java.util.Collections; +import java.util.Optional; + +import static org.apache.tsfile.read.common.type.DoubleType.DOUBLE; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class UserDefineScalarFunctionColumnTransformerTest { + @Test + public void testSelectionSkipsRejectedRowsWithCachedAndUncachedInputs() throws Exception { + for (boolean cached : new boolean[] {false, true}) { + IoTDBLocal local = Mockito.mock(IoTDBLocal.class); + ColumnTransformerBuilder.Context context = + Mockito.mock(ColumnTransformerBuilder.Context.class); + Mockito.when(context.getIoTDBLocalFactory()).thenReturn((s, f, q, d) -> local); + Mockito.when(context.getFragmentInstanceId()).thenReturn("test-fragment"); + Mockito.when(context.getOuterGlobalQueryId()).thenReturn("test-query"); + Mockito.when(context.getOuterQueryDeadlineMs()).thenReturn(Long.MAX_VALUE); + ScalarFunction function = Mockito.mock(ScalarFunction.class); + Mockito.when(function.evaluate(Mockito.any(Record.class), Mockito.eq(local))) + .thenAnswer( + invocation -> { + double value = ((Record) invocation.getArgument(0)).getDouble(0); + assertTrue(value > 0); + return value + 1; + }); + IdentityColumnTransformer input = new IdentityColumnTransformer(DOUBLE, 0); + input.addReferenceCount(); + input.initFromTsBlock( + new TsBlock( + new TimeColumn(2, new long[2]), + new DoubleColumn(2, Optional.empty(), new double[] {-2, 3}))); + if (cached) { + input.tryEvaluate(); + } + UserDefineScalarFunctionTransformer transformer = + new UserDefineScalarFunctionTransformer( + DOUBLE, function, Collections.singletonList(input), null, context); + try { + Column result = transformer.getColumnWithSelection(new boolean[] {false, true}); + assertEquals(2, result.getPositionCount()); + assertTrue(result.isNull(0)); + assertEquals(4.0, result.getDouble(1), 0.0); + Column empty = transformer.getColumnWithSelection(new boolean[] {false, false}); + assertTrue(empty.isNull(0)); + assertTrue(empty.isNull(1)); + Mockito.verify(function, Mockito.times(1)) + .evaluate(Mockito.any(Record.class), Mockito.eq(local)); + } finally { + transformer.close(); + } + } + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/binary/hmac/HmacMd5ColumnTransformerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/binary/hmac/HmacMd5ColumnTransformerTest.java index 65bc3234ff5e6..dd381a0747b20 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/binary/hmac/HmacMd5ColumnTransformerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/binary/hmac/HmacMd5ColumnTransformerTest.java @@ -72,6 +72,7 @@ private byte[] calculateHmacMd5(byte[] data, byte[] key) private ColumnTransformer mockColumnTransformer(Column column) { ColumnTransformer mockTransformer = Mockito.mock(ColumnTransformer.class); Mockito.when(mockTransformer.getColumn()).thenReturn(column); + Mockito.when(mockTransformer.getColumnWithSelection(Mockito.any())).thenReturn(column); // Ensure that tryEvaluate (or other evaluation methods) don't throw exceptions Mockito.doNothing().when(mockTransformer).tryEvaluate(); Mockito.doNothing().when(mockTransformer).clearCache(); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/binary/hmac/HmacSha1ColumnTransformerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/binary/hmac/HmacSha1ColumnTransformerTest.java index e37ad94329cf0..c260856d5440c 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/binary/hmac/HmacSha1ColumnTransformerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/binary/hmac/HmacSha1ColumnTransformerTest.java @@ -72,6 +72,7 @@ private byte[] calculateHmacSha1(byte[] data, byte[] key) private ColumnTransformer mockColumnTransformer(Column column) { ColumnTransformer mockTransformer = Mockito.mock(ColumnTransformer.class); Mockito.when(mockTransformer.getColumn()).thenReturn(column); + Mockito.when(mockTransformer.getColumnWithSelection(Mockito.any())).thenReturn(column); // Ensure that tryEvaluate (or other evaluation methods) don't throw exceptions Mockito.doNothing().when(mockTransformer).tryEvaluate(); Mockito.doNothing().when(mockTransformer).clearCache(); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/binary/hmac/HmacSha256ColumnTransformerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/binary/hmac/HmacSha256ColumnTransformerTest.java index a73d248012868..472a270f2c1c2 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/binary/hmac/HmacSha256ColumnTransformerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/binary/hmac/HmacSha256ColumnTransformerTest.java @@ -72,6 +72,7 @@ private byte[] calculateHmacSha256(byte[] data, byte[] key) private ColumnTransformer mockColumnTransformer(Column column) { ColumnTransformer mockTransformer = Mockito.mock(ColumnTransformer.class); Mockito.when(mockTransformer.getColumn()).thenReturn(column); + Mockito.when(mockTransformer.getColumnWithSelection(Mockito.any())).thenReturn(column); // Ensure that tryEvaluate (or other evaluation methods) don't throw exceptions Mockito.doNothing().when(mockTransformer).tryEvaluate(); Mockito.doNothing().when(mockTransformer).clearCache(); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/binary/hmac/HmacSha512ColumnTransformerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/binary/hmac/HmacSha512ColumnTransformerTest.java index 541faceeebef6..da0e7110a4a86 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/binary/hmac/HmacSha512ColumnTransformerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/binary/hmac/HmacSha512ColumnTransformerTest.java @@ -72,6 +72,7 @@ private byte[] calculateHmacSha512(byte[] data, byte[] key) private ColumnTransformer mockColumnTransformer(Column column) { ColumnTransformer mockTransformer = Mockito.mock(ColumnTransformer.class); Mockito.when(mockTransformer.getColumn()).thenReturn(column); + Mockito.when(mockTransformer.getColumnWithSelection(Mockito.any())).thenReturn(column); // Ensure that tryEvaluate (or other evaluation methods) don't throw exceptions Mockito.doNothing().when(mockTransformer).tryEvaluate(); Mockito.doNothing().when(mockTransformer).clearCache(); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/ternary/LpadColumnTransformerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/ternary/LpadColumnTransformerTest.java index 0bca3e93575b5..44755d61821fa 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/ternary/LpadColumnTransformerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/ternary/LpadColumnTransformerTest.java @@ -44,7 +44,8 @@ private ColumnTransformer mockColumnTransformer(Column column) { // Common mocking for evaluation methods Mockito.doNothing().when(mockTransformer).tryEvaluate(); Mockito.doNothing().when(mockTransformer).clearCache(); - Mockito.doNothing().when(mockTransformer).evaluateWithSelection(Mockito.any()); + Mockito.when(mockTransformer.getColumnWithSelection(Mockito.any())) + .thenAnswer(invocation -> mockTransformer.getColumn()); Mockito.when(mockTransformer.getColumnCachePositionCount()) .thenReturn(column.getPositionCount()); return mockTransformer; diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/ternary/RpadColumnTransformerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/ternary/RpadColumnTransformerTest.java index 7b1250a2491b0..71cadc4f21098 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/ternary/RpadColumnTransformerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/ternary/RpadColumnTransformerTest.java @@ -44,7 +44,8 @@ private ColumnTransformer mockColumnTransformer(Column column) { // Common mocking for evaluation methods Mockito.doNothing().when(mockTransformer).tryEvaluate(); Mockito.doNothing().when(mockTransformer).clearCache(); - Mockito.doNothing().when(mockTransformer).evaluateWithSelection(Mockito.any()); + Mockito.when(mockTransformer.getColumnWithSelection(Mockito.any())) + .thenAnswer(invocation -> mockTransformer.getColumn()); Mockito.when(mockTransformer.getColumnCachePositionCount()) .thenReturn(column.getPositionCount()); return mockTransformer; diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/BlobLengthColumnTransformerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/BlobLengthColumnTransformerTest.java index 6b50738c7184f..09fb5a2cbc984 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/BlobLengthColumnTransformerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/BlobLengthColumnTransformerTest.java @@ -41,7 +41,8 @@ private ColumnTransformer mockChildColumnTransformer(Column column) { Mockito.when(mockColumnTransformer.getColumn()).thenReturn(column); Mockito.doNothing().when(mockColumnTransformer).tryEvaluate(); Mockito.doNothing().when(mockColumnTransformer).clearCache(); - Mockito.doNothing().when(mockColumnTransformer).evaluateWithSelection(Mockito.any()); + Mockito.when(mockColumnTransformer.getColumnWithSelection(Mockito.any())) + .thenAnswer(invocation -> mockColumnTransformer.getColumn()); return mockColumnTransformer; } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/IntegerEncoding/FromBigEndian32ColumnTransformerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/IntegerEncoding/FromBigEndian32ColumnTransformerTest.java index dfbc215a88f8b..a14402023ba30 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/IntegerEncoding/FromBigEndian32ColumnTransformerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/IntegerEncoding/FromBigEndian32ColumnTransformerTest.java @@ -45,7 +45,8 @@ private ColumnTransformer mockChildColumnTransformer(Column column) { Mockito.when(mockColumnTransformer.getColumn()).thenReturn(column); Mockito.doNothing().when(mockColumnTransformer).tryEvaluate(); Mockito.doNothing().when(mockColumnTransformer).clearCache(); - Mockito.doNothing().when(mockColumnTransformer).evaluateWithSelection(Mockito.any()); + Mockito.when(mockColumnTransformer.getColumnWithSelection(Mockito.any())) + .thenAnswer(invocation -> mockColumnTransformer.getColumn()); return mockColumnTransformer; } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/IntegerEncoding/FromBigEndian64ColumnTransformerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/IntegerEncoding/FromBigEndian64ColumnTransformerTest.java index 4cf2a63440b76..25e3a03f6d92b 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/IntegerEncoding/FromBigEndian64ColumnTransformerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/IntegerEncoding/FromBigEndian64ColumnTransformerTest.java @@ -45,7 +45,8 @@ private ColumnTransformer mockChildColumnTransformer(Column column) { Mockito.when(mockColumnTransformer.getColumn()).thenReturn(column); Mockito.doNothing().when(mockColumnTransformer).tryEvaluate(); Mockito.doNothing().when(mockColumnTransformer).clearCache(); - Mockito.doNothing().when(mockColumnTransformer).evaluateWithSelection(Mockito.any()); + Mockito.when(mockColumnTransformer.getColumnWithSelection(Mockito.any())) + .thenAnswer(invocation -> mockColumnTransformer.getColumn()); return mockColumnTransformer; } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/IntegerEncoding/FromIEEE754_32BigEndianColumnTransformerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/IntegerEncoding/FromIEEE754_32BigEndianColumnTransformerTest.java index ebd1d4943352f..42acc65b422e8 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/IntegerEncoding/FromIEEE754_32BigEndianColumnTransformerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/IntegerEncoding/FromIEEE754_32BigEndianColumnTransformerTest.java @@ -45,7 +45,8 @@ private ColumnTransformer mockChildColumnTransformer(Column column) { Mockito.when(mockColumnTransformer.getColumn()).thenReturn(column); Mockito.doNothing().when(mockColumnTransformer).tryEvaluate(); Mockito.doNothing().when(mockColumnTransformer).clearCache(); - Mockito.doNothing().when(mockColumnTransformer).evaluateWithSelection(Mockito.any()); + Mockito.when(mockColumnTransformer.getColumnWithSelection(Mockito.any())) + .thenAnswer(invocation -> mockColumnTransformer.getColumn()); return mockColumnTransformer; } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/IntegerEncoding/FromIEEE754_64BigEndianColumnTransformerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/IntegerEncoding/FromIEEE754_64BigEndianColumnTransformerTest.java index 0d1514e2af82f..a3fcab6fd4436 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/IntegerEncoding/FromIEEE754_64BigEndianColumnTransformerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/IntegerEncoding/FromIEEE754_64BigEndianColumnTransformerTest.java @@ -45,7 +45,8 @@ private ColumnTransformer mockChildColumnTransformer(Column column) { Mockito.when(mockColumnTransformer.getColumn()).thenReturn(column); Mockito.doNothing().when(mockColumnTransformer).tryEvaluate(); Mockito.doNothing().when(mockColumnTransformer).clearCache(); - Mockito.doNothing().when(mockColumnTransformer).evaluateWithSelection(Mockito.any()); + Mockito.when(mockColumnTransformer.getColumnWithSelection(Mockito.any())) + .thenAnswer(invocation -> mockColumnTransformer.getColumn()); return mockColumnTransformer; } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/IntegerEncoding/FromLittleEndian32ColumnTransformerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/IntegerEncoding/FromLittleEndian32ColumnTransformerTest.java index fb99ce5d34dcd..4d07413a79017 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/IntegerEncoding/FromLittleEndian32ColumnTransformerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/IntegerEncoding/FromLittleEndian32ColumnTransformerTest.java @@ -46,7 +46,8 @@ private ColumnTransformer mockChildColumnTransformer(Column column) { Mockito.when(mockColumnTransformer.getColumn()).thenReturn(column); Mockito.doNothing().when(mockColumnTransformer).tryEvaluate(); Mockito.doNothing().when(mockColumnTransformer).clearCache(); - Mockito.doNothing().when(mockColumnTransformer).evaluateWithSelection(Mockito.any()); + Mockito.when(mockColumnTransformer.getColumnWithSelection(Mockito.any())) + .thenAnswer(invocation -> mockColumnTransformer.getColumn()); return mockColumnTransformer; } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/IntegerEncoding/FromLittleEndian64ColumnTransformerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/IntegerEncoding/FromLittleEndian64ColumnTransformerTest.java index 533484563e706..7ff069309fdf0 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/IntegerEncoding/FromLittleEndian64ColumnTransformerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/IntegerEncoding/FromLittleEndian64ColumnTransformerTest.java @@ -46,7 +46,8 @@ private ColumnTransformer mockChildColumnTransformer(Column column) { Mockito.when(mockColumnTransformer.getColumn()).thenReturn(column); Mockito.doNothing().when(mockColumnTransformer).tryEvaluate(); Mockito.doNothing().when(mockColumnTransformer).clearCache(); - Mockito.doNothing().when(mockColumnTransformer).evaluateWithSelection(Mockito.any()); + Mockito.when(mockColumnTransformer.getColumnWithSelection(Mockito.any())) + .thenAnswer(invocation -> mockColumnTransformer.getColumn()); return mockColumnTransformer; } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/IntegerEncoding/ToBigEndian32ColumnTransformerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/IntegerEncoding/ToBigEndian32ColumnTransformerTest.java index e13563e41df23..92a8fab86ea4d 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/IntegerEncoding/ToBigEndian32ColumnTransformerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/IntegerEncoding/ToBigEndian32ColumnTransformerTest.java @@ -42,7 +42,8 @@ private ColumnTransformer mockChildColumnTransformer(Column column) { Mockito.when(mockColumnTransformer.getColumn()).thenReturn(column); Mockito.doNothing().when(mockColumnTransformer).tryEvaluate(); Mockito.doNothing().when(mockColumnTransformer).clearCache(); - Mockito.doNothing().when(mockColumnTransformer).evaluateWithSelection(Mockito.any()); + Mockito.when(mockColumnTransformer.getColumnWithSelection(Mockito.any())) + .thenAnswer(invocation -> mockColumnTransformer.getColumn()); return mockColumnTransformer; } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/IntegerEncoding/ToBigEndian64ColumnTransformerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/IntegerEncoding/ToBigEndian64ColumnTransformerTest.java index 2d15cf4a5c362..24962167a42e2 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/IntegerEncoding/ToBigEndian64ColumnTransformerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/IntegerEncoding/ToBigEndian64ColumnTransformerTest.java @@ -42,7 +42,8 @@ private ColumnTransformer mockChildColumnTransformer(Column column) { Mockito.when(mockColumnTransformer.getColumn()).thenReturn(column); Mockito.doNothing().when(mockColumnTransformer).tryEvaluate(); Mockito.doNothing().when(mockColumnTransformer).clearCache(); - Mockito.doNothing().when(mockColumnTransformer).evaluateWithSelection(Mockito.any()); + Mockito.when(mockColumnTransformer.getColumnWithSelection(Mockito.any())) + .thenAnswer(invocation -> mockColumnTransformer.getColumn()); return mockColumnTransformer; } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/IntegerEncoding/ToIEEE754_32BigEndianColumnTransformerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/IntegerEncoding/ToIEEE754_32BigEndianColumnTransformerTest.java index ffda49dec3bdc..dd767459dbecd 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/IntegerEncoding/ToIEEE754_32BigEndianColumnTransformerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/IntegerEncoding/ToIEEE754_32BigEndianColumnTransformerTest.java @@ -42,7 +42,8 @@ private ColumnTransformer mockChildColumnTransformer(Column column) { Mockito.when(mockColumnTransformer.getColumn()).thenReturn(column); Mockito.doNothing().when(mockColumnTransformer).tryEvaluate(); Mockito.doNothing().when(mockColumnTransformer).clearCache(); - Mockito.doNothing().when(mockColumnTransformer).evaluateWithSelection(Mockito.any()); + Mockito.when(mockColumnTransformer.getColumnWithSelection(Mockito.any())) + .thenAnswer(invocation -> mockColumnTransformer.getColumn()); return mockColumnTransformer; } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/IntegerEncoding/ToIEEE754_64BigEndianColumnTransformerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/IntegerEncoding/ToIEEE754_64BigEndianColumnTransformerTest.java index ded4c9480c8de..c7b017626effd 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/IntegerEncoding/ToIEEE754_64BigEndianColumnTransformerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/IntegerEncoding/ToIEEE754_64BigEndianColumnTransformerTest.java @@ -42,7 +42,8 @@ private ColumnTransformer mockChildColumnTransformer(Column column) { Mockito.when(mockColumnTransformer.getColumn()).thenReturn(column); Mockito.doNothing().when(mockColumnTransformer).tryEvaluate(); Mockito.doNothing().when(mockColumnTransformer).clearCache(); - Mockito.doNothing().when(mockColumnTransformer).evaluateWithSelection(Mockito.any()); + Mockito.when(mockColumnTransformer.getColumnWithSelection(Mockito.any())) + .thenAnswer(invocation -> mockColumnTransformer.getColumn()); return mockColumnTransformer; } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/IntegerEncoding/ToLittleEndian32ColumnTransformerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/IntegerEncoding/ToLittleEndian32ColumnTransformerTest.java index bcee3901b6140..79bab3a3226af 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/IntegerEncoding/ToLittleEndian32ColumnTransformerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/IntegerEncoding/ToLittleEndian32ColumnTransformerTest.java @@ -43,7 +43,8 @@ private ColumnTransformer mockChildColumnTransformer(Column column) { Mockito.when(mockColumnTransformer.getColumn()).thenReturn(column); Mockito.doNothing().when(mockColumnTransformer).tryEvaluate(); Mockito.doNothing().when(mockColumnTransformer).clearCache(); - Mockito.doNothing().when(mockColumnTransformer).evaluateWithSelection(Mockito.any()); + Mockito.when(mockColumnTransformer.getColumnWithSelection(Mockito.any())) + .thenAnswer(invocation -> mockColumnTransformer.getColumn()); return mockColumnTransformer; } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/IntegerEncoding/ToLittleEndian64ColumnTransformerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/IntegerEncoding/ToLittleEndian64ColumnTransformerTest.java index 420eb8ef5673b..764e1b31be897 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/IntegerEncoding/ToLittleEndian64ColumnTransformerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/IntegerEncoding/ToLittleEndian64ColumnTransformerTest.java @@ -43,7 +43,8 @@ private ColumnTransformer mockChildColumnTransformer(Column column) { Mockito.when(mockColumnTransformer.getColumn()).thenReturn(column); Mockito.doNothing().when(mockColumnTransformer).tryEvaluate(); Mockito.doNothing().when(mockColumnTransformer).clearCache(); - Mockito.doNothing().when(mockColumnTransformer).evaluateWithSelection(Mockito.any()); + Mockito.when(mockColumnTransformer.getColumnWithSelection(Mockito.any())) + .thenAnswer(invocation -> mockColumnTransformer.getColumn()); return mockColumnTransformer; } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/ReverseColumnTransformerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/ReverseColumnTransformerTest.java index 03811972f5d19..a018396e92665 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/ReverseColumnTransformerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/ReverseColumnTransformerTest.java @@ -45,7 +45,8 @@ private ColumnTransformer mockChildColumnTransformer(Column column) { Mockito.when(mockColumnTransformer.getColumn()).thenReturn(column); Mockito.doNothing().when(mockColumnTransformer).tryEvaluate(); Mockito.doNothing().when(mockColumnTransformer).clearCache(); - Mockito.doNothing().when(mockColumnTransformer).evaluateWithSelection(Mockito.any()); + Mockito.when(mockColumnTransformer.getColumnWithSelection(Mockito.any())) + .thenAnswer(invocation -> mockColumnTransformer.getColumn()); return mockColumnTransformer; } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/base64Encoding/FromBase32ColumnTransformerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/base64Encoding/FromBase32ColumnTransformerTest.java index 068654977deca..f6b9ef933fe1d 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/base64Encoding/FromBase32ColumnTransformerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/base64Encoding/FromBase32ColumnTransformerTest.java @@ -52,7 +52,8 @@ private ColumnTransformer mockChildColumnTransformer(Column column) { Mockito.when(mockColumnTransformer.getColumn()).thenReturn(column); Mockito.doNothing().when(mockColumnTransformer).tryEvaluate(); Mockito.doNothing().when(mockColumnTransformer).clearCache(); - Mockito.doNothing().when(mockColumnTransformer).evaluateWithSelection(Mockito.any()); + Mockito.when(mockColumnTransformer.getColumnWithSelection(Mockito.any())) + .thenAnswer(invocation -> mockColumnTransformer.getColumn()); return mockColumnTransformer; } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/base64Encoding/FromBase64ColumnTransformerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/base64Encoding/FromBase64ColumnTransformerTest.java index 1135633bb200f..e10cb989a4514 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/base64Encoding/FromBase64ColumnTransformerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/base64Encoding/FromBase64ColumnTransformerTest.java @@ -47,7 +47,8 @@ private ColumnTransformer mockChildColumnTransformer(Column column) { Mockito.when(mockColumnTransformer.getColumn()).thenReturn(column); Mockito.doNothing().when(mockColumnTransformer).tryEvaluate(); Mockito.doNothing().when(mockColumnTransformer).clearCache(); - Mockito.doNothing().when(mockColumnTransformer).evaluateWithSelection(Mockito.any()); + Mockito.when(mockColumnTransformer.getColumnWithSelection(Mockito.any())) + .thenAnswer(invocation -> mockColumnTransformer.getColumn()); return mockColumnTransformer; } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/base64Encoding/FromBase64UrlColumnTransformerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/base64Encoding/FromBase64UrlColumnTransformerTest.java index 83bbe84f4d0b1..f171e7004188b 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/base64Encoding/FromBase64UrlColumnTransformerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/base64Encoding/FromBase64UrlColumnTransformerTest.java @@ -48,7 +48,8 @@ private ColumnTransformer mockChildColumnTransformer(Column column) { Mockito.when(mockColumnTransformer.getColumn()).thenReturn(column); Mockito.doNothing().when(mockColumnTransformer).tryEvaluate(); Mockito.doNothing().when(mockColumnTransformer).clearCache(); - Mockito.doNothing().when(mockColumnTransformer).evaluateWithSelection(Mockito.any()); + Mockito.when(mockColumnTransformer.getColumnWithSelection(Mockito.any())) + .thenAnswer(invocation -> mockColumnTransformer.getColumn()); return mockColumnTransformer; } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/base64Encoding/ToBase32ColumnTransformerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/base64Encoding/ToBase32ColumnTransformerTest.java index d000b22571119..21a1ec62fdf4c 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/base64Encoding/ToBase32ColumnTransformerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/base64Encoding/ToBase32ColumnTransformerTest.java @@ -50,7 +50,8 @@ private ColumnTransformer mockChildColumnTransformer(Column column) { Mockito.when(mockColumnTransformer.getColumn()).thenReturn(column); Mockito.doNothing().when(mockColumnTransformer).tryEvaluate(); Mockito.doNothing().when(mockColumnTransformer).clearCache(); - Mockito.doNothing().when(mockColumnTransformer).evaluateWithSelection(Mockito.any()); + Mockito.when(mockColumnTransformer.getColumnWithSelection(Mockito.any())) + .thenAnswer(invocation -> mockColumnTransformer.getColumn()); return mockColumnTransformer; } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/base64Encoding/ToBase64ColumnTransformerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/base64Encoding/ToBase64ColumnTransformerTest.java index d02f9765b802e..e4fee39dca4d2 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/base64Encoding/ToBase64ColumnTransformerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/base64Encoding/ToBase64ColumnTransformerTest.java @@ -45,7 +45,8 @@ private ColumnTransformer mockChildColumnTransformer(Column column) { Mockito.when(mockColumnTransformer.getColumn()).thenReturn(column); Mockito.doNothing().when(mockColumnTransformer).tryEvaluate(); Mockito.doNothing().when(mockColumnTransformer).clearCache(); - Mockito.doNothing().when(mockColumnTransformer).evaluateWithSelection(Mockito.any()); + Mockito.when(mockColumnTransformer.getColumnWithSelection(Mockito.any())) + .thenAnswer(invocation -> mockColumnTransformer.getColumn()); return mockColumnTransformer; } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/base64Encoding/ToBase64UrlColumnTransformerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/base64Encoding/ToBase64UrlColumnTransformerTest.java index d83fdd7239ffc..5427d60b4ddd1 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/base64Encoding/ToBase64UrlColumnTransformerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/base64Encoding/ToBase64UrlColumnTransformerTest.java @@ -45,7 +45,8 @@ private ColumnTransformer mockChildColumnTransformer(Column column) { Mockito.when(mockColumnTransformer.getColumn()).thenReturn(column); Mockito.doNothing().when(mockColumnTransformer).tryEvaluate(); Mockito.doNothing().when(mockColumnTransformer).clearCache(); - Mockito.doNothing().when(mockColumnTransformer).evaluateWithSelection(Mockito.any()); + Mockito.when(mockColumnTransformer.getColumnWithSelection(Mockito.any())) + .thenAnswer(invocation -> mockColumnTransformer.getColumn()); return mockColumnTransformer; } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/hashing/Crc32ColumnTransformerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/hashing/Crc32ColumnTransformerTest.java index c3829eb04693e..a47b15ec398e6 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/hashing/Crc32ColumnTransformerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/hashing/Crc32ColumnTransformerTest.java @@ -51,7 +51,8 @@ private ColumnTransformer mockChildColumnTransformer(Column column) { Mockito.when(mockColumnTransformer.getColumn()).thenReturn(column); Mockito.doNothing().when(mockColumnTransformer).tryEvaluate(); Mockito.doNothing().when(mockColumnTransformer).clearCache(); - Mockito.doNothing().when(mockColumnTransformer).evaluateWithSelection(Mockito.any()); + Mockito.when(mockColumnTransformer.getColumnWithSelection(Mockito.any())) + .thenAnswer(invocation -> mockColumnTransformer.getColumn()); return mockColumnTransformer; } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/hashing/Md5ColumnTransformerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/hashing/Md5ColumnTransformerTest.java index 7135f334a182e..3c77de2f50ce3 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/hashing/Md5ColumnTransformerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/hashing/Md5ColumnTransformerTest.java @@ -47,7 +47,8 @@ private ColumnTransformer mockChildColumnTransformer(Column column) { Mockito.when(mockColumnTransformer.getColumn()).thenReturn(column); Mockito.doNothing().when(mockColumnTransformer).tryEvaluate(); Mockito.doNothing().when(mockColumnTransformer).clearCache(); - Mockito.doNothing().when(mockColumnTransformer).evaluateWithSelection(Mockito.any()); + Mockito.when(mockColumnTransformer.getColumnWithSelection(Mockito.any())) + .thenAnswer(invocation -> mockColumnTransformer.getColumn()); return mockColumnTransformer; } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/hashing/Murmur3ColumnTransformerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/hashing/Murmur3ColumnTransformerTest.java index 2ce9dba7d8805..6eb78c01111c9 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/hashing/Murmur3ColumnTransformerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/hashing/Murmur3ColumnTransformerTest.java @@ -47,7 +47,8 @@ private ColumnTransformer mockChildColumnTransformer(Column column) { Mockito.when(mockColumnTransformer.getColumn()).thenReturn(column); Mockito.doNothing().when(mockColumnTransformer).tryEvaluate(); Mockito.doNothing().when(mockColumnTransformer).clearCache(); - Mockito.doNothing().when(mockColumnTransformer).evaluateWithSelection(Mockito.any()); + Mockito.when(mockColumnTransformer.getColumnWithSelection(Mockito.any())) + .thenAnswer(invocation -> mockColumnTransformer.getColumn()); return mockColumnTransformer; } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/hashing/Sha1ColumnTransformerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/hashing/Sha1ColumnTransformerTest.java index ef9ae8e222cf4..7b6191071f58e 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/hashing/Sha1ColumnTransformerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/hashing/Sha1ColumnTransformerTest.java @@ -47,7 +47,8 @@ private ColumnTransformer mockChildColumnTransformer(Column column) { Mockito.when(mockColumnTransformer.getColumn()).thenReturn(column); Mockito.doNothing().when(mockColumnTransformer).tryEvaluate(); Mockito.doNothing().when(mockColumnTransformer).clearCache(); - Mockito.doNothing().when(mockColumnTransformer).evaluateWithSelection(Mockito.any()); + Mockito.when(mockColumnTransformer.getColumnWithSelection(Mockito.any())) + .thenAnswer(invocation -> mockColumnTransformer.getColumn()); return mockColumnTransformer; } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/hashing/Sha256ColumnTransformerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/hashing/Sha256ColumnTransformerTest.java index 902693f87afc1..e72de9e6090f7 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/hashing/Sha256ColumnTransformerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/hashing/Sha256ColumnTransformerTest.java @@ -47,7 +47,8 @@ private ColumnTransformer mockChildColumnTransformer(Column column) { Mockito.when(mockColumnTransformer.getColumn()).thenReturn(column); Mockito.doNothing().when(mockColumnTransformer).tryEvaluate(); Mockito.doNothing().when(mockColumnTransformer).clearCache(); - Mockito.doNothing().when(mockColumnTransformer).evaluateWithSelection(Mockito.any()); + Mockito.when(mockColumnTransformer.getColumnWithSelection(Mockito.any())) + .thenAnswer(invocation -> mockColumnTransformer.getColumn()); return mockColumnTransformer; } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/hashing/Sha512ColumnTransformerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/hashing/Sha512ColumnTransformerTest.java index 488b67d02d52d..644af479b381b 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/hashing/Sha512ColumnTransformerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/hashing/Sha512ColumnTransformerTest.java @@ -45,7 +45,8 @@ private ColumnTransformer mockChildColumnTransformer(Column column) { Mockito.when(mockColumnTransformer.getColumn()).thenReturn(column); Mockito.doNothing().when(mockColumnTransformer).tryEvaluate(); Mockito.doNothing().when(mockColumnTransformer).clearCache(); - Mockito.doNothing().when(mockColumnTransformer).evaluateWithSelection(Mockito.any()); + Mockito.when(mockColumnTransformer.getColumnWithSelection(Mockito.any())) + .thenAnswer(invocation -> mockColumnTransformer.getColumn()); return mockColumnTransformer; } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/hashing/SpookyHashV2_32ColumnTransformerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/hashing/SpookyHashV2_32ColumnTransformerTest.java index 258b2db8aad03..19f035ea8facf 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/hashing/SpookyHashV2_32ColumnTransformerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/hashing/SpookyHashV2_32ColumnTransformerTest.java @@ -47,7 +47,8 @@ private ColumnTransformer mockChildColumnTransformer(Column column) { Mockito.when(mockColumnTransformer.getColumn()).thenReturn(column); Mockito.doNothing().when(mockColumnTransformer).tryEvaluate(); Mockito.doNothing().when(mockColumnTransformer).clearCache(); - Mockito.doNothing().when(mockColumnTransformer).evaluateWithSelection(Mockito.any()); + Mockito.when(mockColumnTransformer.getColumnWithSelection(Mockito.any())) + .thenAnswer(invocation -> mockColumnTransformer.getColumn()); return mockColumnTransformer; } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/hashing/SpookyHashV2_64ColumnTransformerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/hashing/SpookyHashV2_64ColumnTransformerTest.java index 5c39e064a4c0c..8aec8f3f19656 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/hashing/SpookyHashV2_64ColumnTransformerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/hashing/SpookyHashV2_64ColumnTransformerTest.java @@ -47,7 +47,8 @@ private ColumnTransformer mockChildColumnTransformer(Column column) { Mockito.when(mockColumnTransformer.getColumn()).thenReturn(column); Mockito.doNothing().when(mockColumnTransformer).tryEvaluate(); Mockito.doNothing().when(mockColumnTransformer).clearCache(); - Mockito.doNothing().when(mockColumnTransformer).evaluateWithSelection(Mockito.any()); + Mockito.when(mockColumnTransformer.getColumnWithSelection(Mockito.any())) + .thenAnswer(invocation -> mockColumnTransformer.getColumn()); return mockColumnTransformer; } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/hashing/XxHash64ColumnTransformerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/hashing/XxHash64ColumnTransformerTest.java index e8a0202919d9e..39af38e18576f 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/hashing/XxHash64ColumnTransformerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/hashing/XxHash64ColumnTransformerTest.java @@ -52,7 +52,8 @@ private ColumnTransformer mockChildColumnTransformer(Column column) { Mockito.when(mockColumnTransformer.getColumn()).thenReturn(column); Mockito.doNothing().when(mockColumnTransformer).tryEvaluate(); Mockito.doNothing().when(mockColumnTransformer).clearCache(); - Mockito.doNothing().when(mockColumnTransformer).evaluateWithSelection(Mockito.any()); + Mockito.when(mockColumnTransformer.getColumnWithSelection(Mockito.any())) + .thenAnswer(invocation -> mockColumnTransformer.getColumn()); return mockColumnTransformer; } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/hexEncoding/FromHexColumnTransformerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/hexEncoding/FromHexColumnTransformerTest.java index 2b636b9095a8b..2933519c5989c 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/hexEncoding/FromHexColumnTransformerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/hexEncoding/FromHexColumnTransformerTest.java @@ -46,7 +46,8 @@ private ColumnTransformer mockChildColumnTransformer(Column column) { Mockito.when(mockColumnTransformer.getColumn()).thenReturn(column); Mockito.doNothing().when(mockColumnTransformer).tryEvaluate(); Mockito.doNothing().when(mockColumnTransformer).clearCache(); - Mockito.doNothing().when(mockColumnTransformer).evaluateWithSelection(Mockito.any()); + Mockito.when(mockColumnTransformer.getColumnWithSelection(Mockito.any())) + .thenAnswer(invocation -> mockColumnTransformer.getColumn()); return mockColumnTransformer; } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/hexEncoding/ToHexColumnTransformerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/hexEncoding/ToHexColumnTransformerTest.java index f846ddce17c57..d2419d027755a 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/hexEncoding/ToHexColumnTransformerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/hexEncoding/ToHexColumnTransformerTest.java @@ -45,7 +45,8 @@ private ColumnTransformer mockChildColumnTransformer(Column column) { Mockito.when(mockColumnTransformer.getColumn()).thenReturn(column); Mockito.doNothing().when(mockColumnTransformer).tryEvaluate(); Mockito.doNothing().when(mockColumnTransformer).clearCache(); - Mockito.doNothing().when(mockColumnTransformer).evaluateWithSelection(Mockito.any()); + Mockito.when(mockColumnTransformer.getColumnWithSelection(Mockito.any())) + .thenAnswer(invocation -> mockColumnTransformer.getColumn()); return mockColumnTransformer; }