diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/aggregation/IoTDBTagAggregationIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/aggregation/IoTDBTagAggregationIT.java index bb7a2645b0c3b..6c7c84ebea0ef 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/aggregation/IoTDBTagAggregationIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/aggregation/IoTDBTagAggregationIT.java @@ -120,15 +120,15 @@ public static void tearDown() throws Exception { public void testAggregateFunctions() { String query = "SELECT COUNT(t), AVG(t), MAX_TIME(t), MIN_TIME(t), MAX_VALUE(t), MIN_VALUE(t), EXTREME(t) FROM root.sg.** GROUP BY TAGS(k1)"; - // Expected result set: + // Expected groups (row order depends on schema traversal): // +----+--------+------------------+-----------+-----------+------------+------------+----------+ // | k1|count(t)| // avg(t)|max_time(t)|min_time(t)|max_value(t)|min_value(t)|extreme(t)| // +----+--------+------------------+-----------+-----------+------------+------------+----------+ - // |k1v2| 4|3.1000000536441803| 10| 1| 5.4| 1.3| - // 5.4| // |k1v1| 6| 2.600000003973643| 10| 1| 6.5| 1.1| // 6.5| + // |k1v2| 4|3.1000000536441803| 10| 1| 5.4| 1.3| + // 5.4| // |NULL| 4| 4.89999994635582| 10| 1| 8.7| 1.6| // 8.7| // +----+--------+------------------+-----------+-----------+------------+------------+----------+ @@ -136,34 +136,39 @@ public void testAggregateFunctions() { Statement statement = connection.createStatement()) { try (ResultSet resultSet = statement.executeQuery(query)) { Assert.assertEquals(8, resultSet.getMetaData().getColumnCount()); - Assert.assertTrue(resultSet.next()); - Assert.assertEquals("k1v2", resultSet.getString("k1")); - Assert.assertEquals(4L, resultSet.getLong("count(t)")); - Assert.assertEquals(3.1D, resultSet.getDouble("avg(t)"), DELTA); - Assert.assertEquals(10L, resultSet.getLong("max_time(t)")); - Assert.assertEquals(1L, resultSet.getLong("min_time(t)")); - Assert.assertEquals(5.4F, resultSet.getFloat("max_value(t)"), DELTA); - Assert.assertEquals(1.3F, resultSet.getFloat("min_value(t)"), DELTA); - Assert.assertEquals(5.4F, resultSet.getFloat("extreme(t)"), DELTA); - Assert.assertTrue(resultSet.next()); - Assert.assertEquals("k1v1", resultSet.getString(1)); - Assert.assertEquals(6L, resultSet.getLong(2)); - Assert.assertEquals(2.6D, resultSet.getDouble(3), DELTA); - Assert.assertEquals(10L, resultSet.getLong(4)); - Assert.assertEquals(1L, resultSet.getLong(5)); - Assert.assertEquals(6.5F, resultSet.getFloat(6), DELTA); - Assert.assertEquals(1.1F, resultSet.getFloat(7), DELTA); - Assert.assertEquals(6.5F, resultSet.getFloat(8), DELTA); - Assert.assertTrue(resultSet.next()); - Assert.assertEquals("NULL", resultSet.getString(1)); - Assert.assertEquals(4L, resultSet.getLong(2)); - Assert.assertEquals(4.9D, resultSet.getDouble(3), DELTA); - Assert.assertEquals(10L, resultSet.getLong(4)); - Assert.assertEquals(1L, resultSet.getLong(5)); - Assert.assertEquals(8.7D, resultSet.getFloat(6), DELTA); - Assert.assertEquals(1.6D, resultSet.getFloat(7), DELTA); - Assert.assertEquals(8.7D, resultSet.getFloat(8), DELTA); - Assert.assertFalse(resultSet.next()); + Set groups = new HashSet<>(); + while (resultSet.next()) { + String tagValue = resultSet.getString("k1"); + Assert.assertTrue(groups.add(tagValue)); + switch (tagValue) { + case "k1v1": + Assert.assertEquals(6L, resultSet.getLong("count(t)")); + Assert.assertEquals(2.6D, resultSet.getDouble("avg(t)"), DELTA); + Assert.assertEquals(6.5F, resultSet.getFloat("max_value(t)"), DELTA); + Assert.assertEquals(1.1F, resultSet.getFloat("min_value(t)"), DELTA); + Assert.assertEquals(6.5F, resultSet.getFloat("extreme(t)"), DELTA); + break; + case "k1v2": + Assert.assertEquals(4L, resultSet.getLong("count(t)")); + Assert.assertEquals(3.1D, resultSet.getDouble("avg(t)"), DELTA); + Assert.assertEquals(5.4F, resultSet.getFloat("max_value(t)"), DELTA); + Assert.assertEquals(1.3F, resultSet.getFloat("min_value(t)"), DELTA); + Assert.assertEquals(5.4F, resultSet.getFloat("extreme(t)"), DELTA); + break; + case "NULL": + Assert.assertEquals(4L, resultSet.getLong("count(t)")); + Assert.assertEquals(4.9D, resultSet.getDouble("avg(t)"), DELTA); + Assert.assertEquals(8.7D, resultSet.getFloat("max_value(t)"), DELTA); + Assert.assertEquals(1.6D, resultSet.getFloat("min_value(t)"), DELTA); + Assert.assertEquals(8.7D, resultSet.getFloat("extreme(t)"), DELTA); + break; + default: + fail("Unexpected tag value: " + tagValue); + } + Assert.assertEquals(10L, resultSet.getLong("max_time(t)")); + Assert.assertEquals(1L, resultSet.getLong("min_time(t)")); + } + Assert.assertEquals(3, groups.size()); } } catch (SQLException e) { e.printStackTrace(); diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/aggregation/IoTDBTimeDurationIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/aggregation/IoTDBTimeDurationIT.java index 75743994b207b..c23b402809a5c 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/aggregation/IoTDBTimeDurationIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/aggregation/IoTDBTimeDurationIT.java @@ -344,7 +344,8 @@ public void testTimeDurationManyColumnOnManyDataRegion() { // group by tags expectedHeader = new String[] {"city", "time_duration(s1)"}; - retArray = new String[] {"Beijing,1677570933,", "Nanjing,1677570938,"}; + // The root.db1 series are encountered first, so the Nanjing group is returned first. + retArray = new String[] {"Nanjing,1677570938,", "Beijing,1677570933,"}; resultSetEqualTest( "select time_duration(s1) from root.db1.**,root.db.** group by tags(city)", expectedHeader, diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/alignbydevice/IoTDBAlignByDeviceWithTemplateAggregationIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/alignbydevice/IoTDBAlignByDeviceWithTemplateAggregationIT.java index 38f2a9616464b..485a1e2b74247 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/alignbydevice/IoTDBAlignByDeviceWithTemplateAggregationIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/alignbydevice/IoTDBAlignByDeviceWithTemplateAggregationIT.java @@ -542,14 +542,14 @@ public void orderByTest() { public void wildCardTest() { String[] expectedHeader = new String[] { - "Device,max_time(s3),max_time(s1),max_time(s2),last_value(s1),last_value(s3),last_value(s1),last_value(s2)" + "Device,max_time(s1),max_time(s2),max_time(s3),last_value(s1),last_value(s1),last_value(s2),last_value(s3)" }; String[] retArray = new String[] { - "root.sg1.d1,1314000000000,1314000000000,1314000000000,13.14,1314,13.14,true,", - "root.sg1.d2,1314000000001,1314000000001,1314000000001,13.15,1315,13.15,false,", - "root.sg1.d3,1314000000002,1314000000002,1314000000002,13.16,1316,13.16,false,", - "root.sg1.d4,1314000000003,1314000000003,1314000000003,13.14,1314,13.14,true,", + "root.sg1.d1,1314000000000,1314000000000,1314000000000,13.14,13.14,true,1314,", + "root.sg1.d2,1314000000001,1314000000001,1314000000001,13.15,13.15,false,1315,", + "root.sg1.d3,1314000000002,1314000000002,1314000000002,13.16,13.16,false,1316,", + "root.sg1.d4,1314000000003,1314000000003,1314000000003,13.14,13.14,true,1314,", }; resultSetEqualTest( "SELECT max_time(*), last_value(s1), last_value(*) FROM root.sg1.** align by device;", @@ -557,10 +557,10 @@ public void wildCardTest() { retArray); retArray = new String[] { - "root.sg2.d1,1314000000000,1314000000000,1314000000000,13.14,1314,13.14,true,", - "root.sg2.d2,1314000000001,1314000000001,1314000000001,13.15,1315,13.15,false,", - "root.sg2.d3,1314000000002,1314000000002,1314000000002,13.16,1316,13.16,false,", - "root.sg2.d4,1314000000003,1314000000003,1314000000003,13.14,1314,13.14,true,", + "root.sg2.d1,1314000000000,1314000000000,1314000000000,13.14,13.14,true,1314,", + "root.sg2.d2,1314000000001,1314000000001,1314000000001,13.15,13.15,false,1315,", + "root.sg2.d3,1314000000002,1314000000002,1314000000002,13.16,13.16,false,1316,", + "root.sg2.d4,1314000000003,1314000000003,1314000000003,13.14,13.14,true,1314,", }; resultSetEqualTest( "SELECT max_time(*), last_value(s1), last_value(*) FROM root.sg2.** align by device;", @@ -569,13 +569,13 @@ public void wildCardTest() { // filter test expectedHeader = - new String[] {"Device,max_time(s1),last_value(s3),last_value(s1),last_value(s2),count(s2)"}; + new String[] {"Device,max_time(s1),last_value(s1),last_value(s2),last_value(s3),count(s2)"}; retArray = new String[] { - "root.sg1.d1,1314000000000,1314,13.14,true,3,", - "root.sg1.d2,1314000000001,1315,13.15,false,4,", - "root.sg1.d3,1314000000002,1316,13.16,false,3,", - "root.sg1.d4,1314000000003,1314,13.14,true,4,", + "root.sg1.d1,1314000000000,13.14,true,1314,3,", + "root.sg1.d2,1314000000001,13.15,false,1315,4,", + "root.sg1.d3,1314000000002,13.16,false,1316,3,", + "root.sg1.d4,1314000000003,13.14,true,1314,4,", }; resultSetEqualTest( "SELECT max_time(s1), last_value(*), count(s2) FROM root.sg1.** where s3>1 align by device;", @@ -583,10 +583,10 @@ public void wildCardTest() { retArray); retArray = new String[] { - "root.sg2.d1,1314000000000,1314,13.14,true,3,", - "root.sg2.d2,1314000000001,1315,13.15,false,4,", - "root.sg2.d3,1314000000002,1316,13.16,false,3,", - "root.sg2.d4,1314000000003,1314,13.14,true,4,", + "root.sg2.d1,1314000000000,13.14,true,1314,3,", + "root.sg2.d2,1314000000001,13.15,false,1315,4,", + "root.sg2.d3,1314000000002,13.16,false,1316,3,", + "root.sg2.d4,1314000000003,13.14,true,1314,4,", }; resultSetEqualTest( "SELECT max_time(s1), last_value(*), count(s2) FROM root.sg2.** where s3>1 align by device;", @@ -594,16 +594,16 @@ public void wildCardTest() { retArray); // sliding window - expectedHeader = new String[] {"Time,Device,last_value(s1),last_value(s2)"}; + expectedHeader = new String[] {"Time,Device,last_value(s2),last_value(s3)"}; retArray = new String[] { - "1,root.sg1.d1,2.2,false,", - "1,root.sg1.d2,22.2,false,", - "3,root.sg1.d2,50.0,false,", - "5,root.sg1.d2,50.0,false,", - "7,root.sg1.d3,8.8,false,", - "3,root.sg1.d4,5555.5,false,", - "5,root.sg1.d4,5555.5,false,", + "1,root.sg1.d1,false,2,", + "1,root.sg1.d2,false,22,", + "3,root.sg1.d2,false,5,", + "5,root.sg1.d2,false,5,", + "7,root.sg1.d3,false,4,", + "3,root.sg1.d4,false,5555,", + "5,root.sg1.d4,false,5555,", }; resultSetEqualTest( "SELECT last_value(*) FROM root.sg1.** where s3+1=1316 or s2=false group by ([1,10),3ms,2ms) having avg(s1)>0 soffset 1 slimit 2 align by device;", @@ -611,13 +611,13 @@ public void wildCardTest() { retArray); retArray = new String[] { - "1,root.sg2.d1,2.2,false,", - "1,root.sg2.d2,22.2,false,", - "3,root.sg2.d2,50.0,false,", - "5,root.sg2.d2,50.0,false,", - "7,root.sg2.d3,8.8,false,", - "3,root.sg2.d4,5555.5,false,", - "5,root.sg2.d4,5555.5,false,", + "1,root.sg2.d1,false,2,", + "1,root.sg2.d2,false,22,", + "3,root.sg2.d2,false,5,", + "5,root.sg2.d2,false,5,", + "7,root.sg2.d3,false,4,", + "3,root.sg2.d4,false,5555,", + "5,root.sg2.d4,false,5555,", }; resultSetEqualTest( "SELECT last_value(*) FROM root.sg2.** where s3+1=1316 or s2=false group by ([1,10),3ms,2ms) having avg(s1)>0 soffset 1 slimit 2 align by device;", @@ -627,11 +627,11 @@ public void wildCardTest() { // having expectedHeader = new String[] { - "Device,last_value(s3),last_value(s1),last_value(s2),first_value(s3),first_value(s1),first_value(s2)" + "Device,last_value(s1),last_value(s2),last_value(s3),first_value(s1),first_value(s2),first_value(s3)" }; retArray = new String[] { - "root.sg1.d2,1315,13.15,false,11,11.1,false,", + "root.sg1.d2,13.15,false,1315,11.1,false,11,", }; resultSetEqualTest( "SELECT last_value(*), first_value(*) FROM root.sg1.** where s2=false having count(s3+s1) > 2 align by device;", @@ -639,7 +639,7 @@ public void wildCardTest() { retArray); retArray = new String[] { - "root.sg2.d2,1315,13.15,false,11,11.1,false,", + "root.sg2.d2,13.15,false,1315,11.1,false,11,", }; resultSetEqualTest( "SELECT last_value(*), first_value(*) FROM root.sg2.** where s2=false having count(s3+s1) > 2 align by device;", @@ -647,7 +647,7 @@ public void wildCardTest() { retArray); // not supported expression: agg1(*)+agg2(*), agg(*)/2 - expectedHeader = new String[] {"Device,count(s3) + 1,count(s1) + 1,count(s2) + 1"}; + expectedHeader = new String[] {"Device,count(s1) + 1,count(s2) + 1,count(s3) + 1"}; retArray = new String[] { "root.sg1.d2,5.0,5.0,5.0,", diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/alignbydevice/IoTDBAlignByDeviceWithTemplateIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/alignbydevice/IoTDBAlignByDeviceWithTemplateIT.java index 82758290dbbc8..f88f0894c3233 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/alignbydevice/IoTDBAlignByDeviceWithTemplateIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/alignbydevice/IoTDBAlignByDeviceWithTemplateIT.java @@ -75,10 +75,10 @@ public static void tearDown() { @Test public void singleDeviceTest() { - expectedHeader = new String[] {"Time,Device,s3,s1,s2"}; + expectedHeader = new String[] {"Time,Device,s1,s2,s3"}; retArray = new String[] { - "1,root.sg1.d1,1,1.1,false,", + "1,root.sg1.d1,1.1,false,1,", }; resultSetEqualTest( "SELECT * FROM root.sg1.d1 order by time desc offset 1 limit 1 ALIGN BY DEVICE;", @@ -86,7 +86,7 @@ public void singleDeviceTest() { retArray); retArray = new String[] { - "1,root.sg2.d1,1,1.1,false,", + "1,root.sg2.d1,1.1,false,1,", }; resultSetEqualTest( "SELECT * FROM root.sg2.d1 order by time desc offset 1 limit 1 ALIGN BY DEVICE;", @@ -97,65 +97,65 @@ public void singleDeviceTest() { @Test public void selectWildcardNoFilterTest() { // 1. order by device - expectedHeader = new String[] {"Time,Device,s3,s1,s2"}; + expectedHeader = new String[] {"Time,Device,s1,s2,s3"}; retArray = new String[] { - "1,root.sg1.d1,1,1.1,false,", - "2,root.sg1.d1,2,2.2,false,", - "1,root.sg1.d2,11,11.1,false,", - "2,root.sg1.d2,22,22.2,false,", - "1,root.sg1.d3,null,111.1,true,", - "4,root.sg1.d3,44,444.4,true,", - "1,root.sg1.d4,1111,1111.1,true,", - "5,root.sg1.d4,5555,5555.5,false,", + "1,root.sg1.d1,1.1,false,1,", + "2,root.sg1.d1,2.2,false,2,", + "1,root.sg1.d2,11.1,false,11,", + "2,root.sg1.d2,22.2,false,22,", + "1,root.sg1.d3,111.1,true,null,", + "4,root.sg1.d3,444.4,true,44,", + "1,root.sg1.d4,1111.1,true,1111,", + "5,root.sg1.d4,5555.5,false,5555,", }; resultSetEqualTest("SELECT * FROM root.sg1.** ALIGN BY DEVICE;", expectedHeader, retArray); retArray = new String[] { - "1,root.sg2.d1,1,1.1,false,", - "2,root.sg2.d1,2,2.2,false,", - "1,root.sg2.d2,11,11.1,false,", - "2,root.sg2.d2,22,22.2,false,", - "1,root.sg2.d3,null,111.1,true,", - "4,root.sg2.d3,44,444.4,true,", - "1,root.sg2.d4,1111,1111.1,true,", - "5,root.sg2.d4,5555,5555.5,false,", + "1,root.sg2.d1,1.1,false,1,", + "2,root.sg2.d1,2.2,false,2,", + "1,root.sg2.d2,11.1,false,11,", + "2,root.sg2.d2,22.2,false,22,", + "1,root.sg2.d3,111.1,true,null,", + "4,root.sg2.d3,444.4,true,44,", + "1,root.sg2.d4,1111.1,true,1111,", + "5,root.sg2.d4,5555.5,false,5555,", }; resultSetEqualTest("SELECT * FROM root.sg2.** ALIGN BY DEVICE;", expectedHeader, retArray); - expectedHeader = new String[] {"Time,Device,s3,s1,s2,s1"}; + expectedHeader = new String[] {"Time,Device,s1,s2,s3,s1"}; retArray = new String[] { - "1,root.sg1.d1,1,1.1,false,1.1,", - "2,root.sg1.d1,2,2.2,false,2.2,", - "1,root.sg1.d2,11,11.1,false,11.1,", - "2,root.sg1.d2,22,22.2,false,22.2,", - "1,root.sg1.d3,null,111.1,true,111.1,", - "4,root.sg1.d3,44,444.4,true,444.4,", - "1,root.sg1.d4,1111,1111.1,true,1111.1,", - "5,root.sg1.d4,5555,5555.5,false,5555.5,", + "1,root.sg1.d1,1.1,false,1,1.1,", + "2,root.sg1.d1,2.2,false,2,2.2,", + "1,root.sg1.d2,11.1,false,11,11.1,", + "2,root.sg1.d2,22.2,false,22,22.2,", + "1,root.sg1.d3,111.1,true,null,111.1,", + "4,root.sg1.d3,444.4,true,44,444.4,", + "1,root.sg1.d4,1111.1,true,1111,1111.1,", + "5,root.sg1.d4,5555.5,false,5555,5555.5,", }; resultSetEqualTest("SELECT *, s1 FROM root.sg1.** ALIGN BY DEVICE;", expectedHeader, retArray); retArray = new String[] { - "1,root.sg2.d1,1,1.1,false,1.1,", - "2,root.sg2.d1,2,2.2,false,2.2,", - "1,root.sg2.d2,11,11.1,false,11.1,", - "2,root.sg2.d2,22,22.2,false,22.2,", - "1,root.sg2.d3,null,111.1,true,111.1,", - "4,root.sg2.d3,44,444.4,true,444.4,", - "1,root.sg2.d4,1111,1111.1,true,1111.1,", - "5,root.sg2.d4,5555,5555.5,false,5555.5,", + "1,root.sg2.d1,1.1,false,1,1.1,", + "2,root.sg2.d1,2.2,false,2,2.2,", + "1,root.sg2.d2,11.1,false,11,11.1,", + "2,root.sg2.d2,22.2,false,22,22.2,", + "1,root.sg2.d3,111.1,true,null,111.1,", + "4,root.sg2.d3,444.4,true,44,444.4,", + "1,root.sg2.d4,1111.1,true,1111,1111.1,", + "5,root.sg2.d4,5555.5,false,5555,5555.5,", }; resultSetEqualTest("SELECT *, s1 FROM root.sg2.** ALIGN BY DEVICE;", expectedHeader, retArray); - expectedHeader = new String[] {"Time,Device,s3,s1,s2"}; + expectedHeader = new String[] {"Time,Device,s1,s2,s3"}; retArray = new String[] { - "1,root.sg1.d1,1,1.1,false,", - "2,root.sg1.d1,2,2.2,false,", - "1,root.sg1.d2,11,11.1,false,", - "2,root.sg1.d2,22,22.2,false,", + "1,root.sg1.d1,1.1,false,1,", + "2,root.sg1.d1,2.2,false,2,", + "1,root.sg1.d2,11.1,false,11,", + "2,root.sg1.d2,22.2,false,22,", }; resultSetEqualTest( "SELECT * FROM root.sg1.d1,root.sg1.d2,root.sg1.d6 ALIGN BY DEVICE;", @@ -163,10 +163,10 @@ public void selectWildcardNoFilterTest() { retArray); retArray = new String[] { - "1,root.sg2.d1,1,1.1,false,", - "2,root.sg2.d1,2,2.2,false,", - "1,root.sg2.d2,11,11.1,false,", - "2,root.sg2.d2,22,22.2,false,", + "1,root.sg2.d1,1.1,false,1,", + "2,root.sg2.d1,2.2,false,2,", + "1,root.sg2.d2,11.1,false,11,", + "2,root.sg2.d2,22.2,false,22,", }; resultSetEqualTest( "SELECT * FROM root.sg2.d1,root.sg2.d2,root.sg2.d6 ALIGN BY DEVICE;", @@ -174,17 +174,17 @@ public void selectWildcardNoFilterTest() { retArray); // 2. order by device + limit/offset - expectedHeader = new String[] {"Time,Device,s3,s1,s2"}; + expectedHeader = new String[] {"Time,Device,s1,s2,s3"}; retArray = new String[] { - "2,root.sg1.d1,2,2.2,false,", "1,root.sg1.d2,11,11.1,false,", + "2,root.sg1.d1,2.2,false,2,", "1,root.sg1.d2,11.1,false,11,", }; resultSetEqualTest( "SELECT * FROM root.sg1.** OFFSET 1 LIMIT 2 ALIGN BY DEVICE;", expectedHeader, retArray); retArray = new String[] { - "2,root.sg2.d1,2,2.2,false,", "1,root.sg2.d2,11,11.1,false,", + "2,root.sg2.d1,2.2,false,2,", "1,root.sg2.d2,11.1,false,11,", }; resultSetEqualTest( "SELECT * FROM root.sg2.** OFFSET 1 LIMIT 2 ALIGN BY DEVICE;", expectedHeader, retArray); @@ -192,28 +192,28 @@ public void selectWildcardNoFilterTest() { // 3. order by time retArray = new String[] { - "5,root.sg1.d4,5555,5555.5,false,", - "4,root.sg1.d3,44,444.4,true,", - "2,root.sg1.d1,2,2.2,false,", - "2,root.sg1.d2,22,22.2,false,", - "1,root.sg1.d1,1,1.1,false,", - "1,root.sg1.d2,11,11.1,false,", - "1,root.sg1.d3,null,111.1,true,", - "1,root.sg1.d4,1111,1111.1,true,", + "5,root.sg1.d4,5555.5,false,5555,", + "4,root.sg1.d3,444.4,true,44,", + "2,root.sg1.d1,2.2,false,2,", + "2,root.sg1.d2,22.2,false,22,", + "1,root.sg1.d1,1.1,false,1,", + "1,root.sg1.d2,11.1,false,11,", + "1,root.sg1.d3,111.1,true,null,", + "1,root.sg1.d4,1111.1,true,1111,", }; resultSetEqualTest( "SELECT * FROM root.sg1.** ORDER BY TIME DESC ALIGN BY DEVICE;", expectedHeader, retArray); retArray = new String[] { - "5,root.sg2.d4,5555,5555.5,false,", - "4,root.sg2.d3,44,444.4,true,", - "2,root.sg2.d1,2,2.2,false,", - "2,root.sg2.d2,22,22.2,false,", - "1,root.sg2.d1,1,1.1,false,", - "1,root.sg2.d2,11,11.1,false,", - "1,root.sg2.d3,null,111.1,true,", - "1,root.sg2.d4,1111,1111.1,true,", + "5,root.sg2.d4,5555.5,false,5555,", + "4,root.sg2.d3,444.4,true,44,", + "2,root.sg2.d1,2.2,false,2,", + "2,root.sg2.d2,22.2,false,22,", + "1,root.sg2.d1,1.1,false,1,", + "1,root.sg2.d2,11.1,false,11,", + "1,root.sg2.d3,111.1,true,null,", + "1,root.sg2.d4,1111.1,true,1111,", }; resultSetEqualTest( "SELECT * FROM root.sg2.** ORDER BY TIME DESC ALIGN BY DEVICE;", expectedHeader, retArray); @@ -221,8 +221,8 @@ public void selectWildcardNoFilterTest() { // 4. order by time + limit/offset retArray = new String[] { - "5,root.sg1.d4,5555,5555.5,false,", "4,root.sg1.d3,44,444.4,true,", - "2,root.sg1.d1,2,2.2,false,", "2,root.sg1.d2,22,22.2,false,", + "5,root.sg1.d4,5555.5,false,5555,", "4,root.sg1.d3,444.4,true,44,", + "2,root.sg1.d1,2.2,false,2,", "2,root.sg1.d2,22.2,false,22,", }; resultSetEqualTest( "SELECT * FROM root.sg1.** ORDER BY TIME DESC LIMIT 4 ALIGN BY DEVICE;", @@ -231,8 +231,8 @@ public void selectWildcardNoFilterTest() { retArray = new String[] { - "5,root.sg2.d4,5555,5555.5,false,", "4,root.sg2.d3,44,444.4,true,", - "2,root.sg2.d1,2,2.2,false,", "2,root.sg2.d2,22,22.2,false,", + "5,root.sg2.d4,5555.5,false,5555,", "4,root.sg2.d3,444.4,true,44,", + "2,root.sg2.d1,2.2,false,2,", "2,root.sg2.d2,22.2,false,22,", }; resultSetEqualTest( "SELECT * FROM root.sg2.** ORDER BY TIME DESC LIMIT 4 ALIGN BY DEVICE;", @@ -351,13 +351,13 @@ public void selectMeasurementNoFilterTest() { @Test public void selectWildcardWithFilterOrderByTimeTest() { // 1. order by time + time filter - String[] expectedHeader = new String[] {"Time,Device,s3,s1,s2"}; + String[] expectedHeader = new String[] {"Time,Device,s1,s2,s3"}; String[] retArray = new String[] { - "4,root.sg1.d3,44,444.4,true,", - "2,root.sg1.d1,2,2.2,false,", - "2,root.sg1.d2,22,22.2,false,", - "1,root.sg1.d1,1,1.1,false,", + "4,root.sg1.d3,444.4,true,44,", + "2,root.sg1.d1,2.2,false,2,", + "2,root.sg1.d2,22.2,false,22,", + "1,root.sg1.d1,1.1,false,1,", }; resultSetEqualTest( "SELECT * FROM root.sg1.** WHERE time < 5 ORDER BY TIME DESC LIMIT 4 ALIGN BY DEVICE;", @@ -366,10 +366,10 @@ public void selectWildcardWithFilterOrderByTimeTest() { retArray = new String[] { - "4,root.sg2.d3,44,444.4,true,", - "2,root.sg2.d1,2,2.2,false,", - "2,root.sg2.d2,22,22.2,false,", - "1,root.sg2.d1,1,1.1,false,", + "4,root.sg2.d3,444.4,true,44,", + "2,root.sg2.d1,2.2,false,2,", + "2,root.sg2.d2,22.2,false,22,", + "1,root.sg2.d1,1.1,false,1,", }; resultSetEqualTest( "SELECT * FROM root.sg2.** WHERE time < 5 ORDER BY TIME DESC LIMIT 4 ALIGN BY DEVICE;", @@ -379,7 +379,7 @@ public void selectWildcardWithFilterOrderByTimeTest() { // 2. order by time + time filter + value filter retArray = new String[] { - "4,root.sg1.d3,44,444.4,true,", "2,root.sg1.d2,22,22.2,false,", + "4,root.sg1.d3,444.4,true,44,", "2,root.sg1.d2,22.2,false,22,", }; resultSetEqualTest( "SELECT * FROM root.sg1.** where time > 1 and time < 5 and s3>=11 and s3<=1111 and s1 != 11.1 " @@ -389,7 +389,7 @@ public void selectWildcardWithFilterOrderByTimeTest() { retArray = new String[] { - "4,root.sg2.d3,44,444.4,true,", "2,root.sg2.d2,22,22.2,false,", + "4,root.sg2.d3,444.4,true,44,", "2,root.sg2.d2,22.2,false,22,", }; resultSetEqualTest( "SELECT * FROM root.sg2.** where time > 1 and time < 5 and s3>=11 and s3<=1111 and s1 != 11.1 " @@ -410,7 +410,7 @@ public void selectWildcardWithFilterOrderByTimeTest() { retArray = new String[] { - "4,root.sg1.d3,44,444.4,true,", "2,root.sg1.d2,22,22.2,false,", + "4,root.sg1.d3,444.4,true,44,", "2,root.sg1.d2,22.2,false,22,", }; resultSetEqualTest( "SELECT * FROM root.sg1.** WHERE s_null > 1 or " @@ -419,7 +419,7 @@ public void selectWildcardWithFilterOrderByTimeTest() { retArray); retArray = new String[] { - "4,root.sg2.d3,44,444.4,true,", "2,root.sg2.d2,22,22.2,false,", + "4,root.sg2.d3,444.4,true,44,", "2,root.sg2.d2,22.2,false,22,", }; resultSetEqualTest( "SELECT * FROM root.sg2.** WHERE s_null > 1 or " @@ -431,13 +431,13 @@ public void selectWildcardWithFilterOrderByTimeTest() { @Test public void selectWildcardWithFilterOrderByDeviceTest() { // 1. order by device + time filter - String[] expectedHeader = new String[] {"Time,Device,s3,s1,s2"}; + String[] expectedHeader = new String[] {"Time,Device,s1,s2,s3"}; String[] retArray = new String[] { - "1,root.sg1.d4,1111,1111.1,true,", - "1,root.sg1.d3,null,111.1,true,", - "4,root.sg1.d3,44,444.4,true,", - "1,root.sg1.d2,11,11.1,false,", + "1,root.sg1.d4,1111.1,true,1111,", + "1,root.sg1.d3,111.1,true,null,", + "4,root.sg1.d3,444.4,true,44,", + "1,root.sg1.d2,11.1,false,11,", }; resultSetEqualTest( "SELECT * FROM root.sg1.** WHERE time < 5 ORDER BY DEVICE DESC LIMIT 4 ALIGN BY DEVICE;", @@ -446,10 +446,10 @@ public void selectWildcardWithFilterOrderByDeviceTest() { retArray = new String[] { - "1,root.sg2.d4,1111,1111.1,true,", - "1,root.sg2.d3,null,111.1,true,", - "4,root.sg2.d3,44,444.4,true,", - "1,root.sg2.d2,11,11.1,false,", + "1,root.sg2.d4,1111.1,true,1111,", + "1,root.sg2.d3,111.1,true,null,", + "4,root.sg2.d3,444.4,true,44,", + "1,root.sg2.d2,11.1,false,11,", }; resultSetEqualTest( "SELECT * FROM root.sg2.** WHERE time < 5 ORDER BY DEVICE DESC LIMIT 4 ALIGN BY DEVICE;", @@ -459,7 +459,7 @@ public void selectWildcardWithFilterOrderByDeviceTest() { // 2. order by device + time filter + value filter retArray = new String[] { - "4,root.sg1.d3,44,444.4,true,", "2,root.sg1.d2,22,22.2,false,", + "4,root.sg1.d3,444.4,true,44,", "2,root.sg1.d2,22.2,false,22,", }; resultSetEqualTest( "SELECT * FROM root.sg1.** where time > 1 and time < 5 and s3>=11 and s3<=1111 and s1 != 11.1 " @@ -469,7 +469,7 @@ public void selectWildcardWithFilterOrderByDeviceTest() { retArray = new String[] { - "2,root.sg2.d2,22,22.2,false,", "4,root.sg2.d3,44,444.4,true,", + "2,root.sg2.d2,22.2,false,22,", "4,root.sg2.d3,444.4,true,44,", }; resultSetEqualTest( "SELECT * FROM root.sg2.** where time > 1 and time < 5 and s3>=11 and s3<=1111 and s1 != 11.1 " @@ -490,7 +490,7 @@ public void selectWildcardWithFilterOrderByDeviceTest() { retArray = new String[] { - "4,root.sg1.d3,44,444.4,true,", "2,root.sg1.d2,22,22.2,false,", + "4,root.sg1.d3,444.4,true,44,", "2,root.sg1.d2,22.2,false,22,", }; resultSetEqualTest( "SELECT * FROM root.sg1.** WHERE s_null > 1 or " @@ -499,7 +499,7 @@ public void selectWildcardWithFilterOrderByDeviceTest() { retArray); retArray = new String[] { - "4,root.sg2.d3,44,444.4,true,", "2,root.sg2.d2,22,22.2,false,", + "4,root.sg2.d3,444.4,true,44,", "2,root.sg2.d2,22.2,false,22,", }; resultSetEqualTest( "SELECT * FROM root.sg2.** WHERE s_null > 1 or " @@ -741,17 +741,17 @@ public void aliasTest() { public void orderByExpressionTest() { // order by expression is not supported temporarily // 1. order by basic measurement - String[] expectedHeader = new String[] {"Time,Device,s3,s1,s2"}; + String[] expectedHeader = new String[] {"Time,Device,s1,s2,s3"}; String[] retArray = new String[] { - "5,root.sg1.d4,5555,5555.5,false,", - "2,root.sg1.d2,22,22.2,false,", - "1,root.sg1.d2,11,11.1,false,", - "2,root.sg1.d1,2,2.2,false,", - "1,root.sg1.d1,1,1.1,false,", - "1,root.sg1.d4,1111,1111.1,true,", - "4,root.sg1.d3,44,444.4,true,", - "1,root.sg1.d3,null,111.1,true,", + "5,root.sg1.d4,5555.5,false,5555,", + "2,root.sg1.d2,22.2,false,22,", + "1,root.sg1.d2,11.1,false,11,", + "2,root.sg1.d1,2.2,false,2,", + "1,root.sg1.d1,1.1,false,1,", + "1,root.sg1.d4,1111.1,true,1111,", + "4,root.sg1.d3,444.4,true,44,", + "1,root.sg1.d3,111.1,true,null,", }; resultSetEqualTest( "SELECT * FROM root.sg1.** order by s2 asc, s1 desc ALIGN BY DEVICE;", @@ -760,14 +760,14 @@ public void orderByExpressionTest() { retArray = new String[] { - "5,root.sg2.d4,5555,5555.5,false,", - "2,root.sg2.d2,22,22.2,false,", - "1,root.sg2.d2,11,11.1,false,", - "2,root.sg2.d1,2,2.2,false,", - "1,root.sg2.d1,1,1.1,false,", - "1,root.sg2.d4,1111,1111.1,true,", - "4,root.sg2.d3,44,444.4,true,", - "1,root.sg2.d3,null,111.1,true,", + "5,root.sg2.d4,5555.5,false,5555,", + "2,root.sg2.d2,22.2,false,22,", + "1,root.sg2.d2,11.1,false,11,", + "2,root.sg2.d1,2.2,false,2,", + "1,root.sg2.d1,1.1,false,1,", + "1,root.sg2.d4,1111.1,true,1111,", + "4,root.sg2.d3,444.4,true,44,", + "1,root.sg2.d3,111.1,true,null,", }; resultSetEqualTest( "SELECT * FROM root.sg2.** order by s2 asc, s1 desc ALIGN BY DEVICE;", @@ -844,17 +844,17 @@ public void orderByExpressionTest() { @Test public void templateInvalidTest() { // 1. non align by device query - String[] expectedHeader = new String[] {"Time,root.sg1.d4.s3,root.sg1.d4.s1,root.sg1.d4.s2"}; + String[] expectedHeader = new String[] {"Time,root.sg1.d4.s1,root.sg1.d4.s2,root.sg1.d4.s3"}; String[] retArray = new String[] { - "1,1111,1111.1,true,", "5,5555,5555.5,false,", + "1,1111.1,true,1111,", "5,5555.5,false,5555,", }; resultSetEqualTest("SELECT * FROM root.sg1.** slimit 3;", expectedHeader, retArray); - expectedHeader = new String[] {"Time,root.sg2.d4.s3,root.sg2.d4.s1,root.sg2.d4.s2"}; + expectedHeader = new String[] {"Time,root.sg2.d4.s1,root.sg2.d4.s2,root.sg2.d4.s3"}; retArray = new String[] { - "1,1111,1111.1,true,", "5,5555,5555.5,false,", + "1,1111.1,true,1111,", "5,5555.5,false,5555,", }; resultSetEqualTest("SELECT * FROM root.sg2.** slimit 3;", expectedHeader, retArray); @@ -886,45 +886,45 @@ public void templateInvalidTest() { @Test public void sLimitOffsetTest() { // 1. original - String[] expectedHeader = new String[] {"Time,Device,s1"}; + String[] expectedHeader = new String[] {"Time,Device,s2"}; String[] retArray = new String[] { - "1,root.sg1.d1,1.1,", - "2,root.sg1.d1,2.2,", - "1,root.sg1.d2,11.1,", - "2,root.sg1.d2,22.2,", - "1,root.sg1.d3,111.1,", - "4,root.sg1.d3,444.4,", - "1,root.sg1.d4,1111.1,", - "5,root.sg1.d4,5555.5,", + "1,root.sg1.d1,false,", + "2,root.sg1.d1,false,", + "1,root.sg1.d2,false,", + "2,root.sg1.d2,false,", + "1,root.sg1.d3,true,", + "4,root.sg1.d3,true,", + "1,root.sg1.d4,true,", + "5,root.sg1.d4,false,", }; resultSetEqualTest( "SELECT * FROM root.sg1.** slimit 1 soffset 1 ALIGN BY DEVICE;", expectedHeader, retArray); retArray = new String[] { - "1,root.sg2.d1,1.1,", - "2,root.sg2.d1,2.2,", - "1,root.sg2.d2,11.1,", - "2,root.sg2.d2,22.2,", - "1,root.sg2.d3,111.1,", - "4,root.sg2.d3,444.4,", - "1,root.sg2.d4,1111.1,", - "5,root.sg2.d4,5555.5,", + "1,root.sg2.d1,false,", + "2,root.sg2.d1,false,", + "1,root.sg2.d2,false,", + "2,root.sg2.d2,false,", + "1,root.sg2.d3,true,", + "4,root.sg2.d3,true,", + "1,root.sg2.d4,true,", + "5,root.sg2.d4,false,", }; resultSetEqualTest( "SELECT * FROM root.sg2.** slimit 1 soffset 1 ALIGN BY DEVICE;", expectedHeader, retArray); - expectedHeader = new String[] {"Time,Device,s1,s2"}; + expectedHeader = new String[] {"Time,Device,s2,s3"}; retArray = new String[] { - "1,root.sg1.d1,1.1,false,", - "2,root.sg1.d1,2.2,false,", - "1,root.sg1.d2,11.1,false,", - "2,root.sg1.d2,22.2,false,", - "1,root.sg1.d3,111.1,true,", - "4,root.sg1.d3,444.4,true,", - "1,root.sg1.d4,1111.1,true,", - "5,root.sg1.d4,5555.5,false,", + "1,root.sg1.d1,false,1,", + "2,root.sg1.d1,false,2,", + "1,root.sg1.d2,false,11,", + "2,root.sg1.d2,false,22,", + "1,root.sg1.d3,true,null,", + "4,root.sg1.d3,true,44,", + "1,root.sg1.d4,true,1111,", + "5,root.sg1.d4,false,5555,", }; resultSetEqualTest( "SELECT *, s1 FROM root.sg1.** slimit 2 soffset 1 ALIGN BY DEVICE;", @@ -932,24 +932,27 @@ public void sLimitOffsetTest() { retArray); retArray = new String[] { - "1,root.sg2.d1,1.1,false,", - "2,root.sg2.d1,2.2,false,", - "1,root.sg2.d2,11.1,false,", - "2,root.sg2.d2,22.2,false,", - "1,root.sg2.d3,111.1,true,", - "4,root.sg2.d3,444.4,true,", - "1,root.sg2.d4,1111.1,true,", - "5,root.sg2.d4,5555.5,false,", + "1,root.sg2.d1,false,1,", + "2,root.sg2.d1,false,2,", + "1,root.sg2.d2,false,11,", + "2,root.sg2.d2,false,22,", + "1,root.sg2.d3,true,null,", + "4,root.sg2.d3,true,44,", + "1,root.sg2.d4,true,1111,", + "5,root.sg2.d4,false,5555,", }; resultSetEqualTest( "SELECT *, s1 FROM root.sg2.** slimit 2 soffset 1 ALIGN BY DEVICE;", expectedHeader, retArray); - expectedHeader = new String[] {"Time,Device,s1"}; + expectedHeader = new String[] {"Time,Device,s2"}; retArray = new String[] { - "1,root.sg1.d1,1.1,", "2,root.sg1.d1,2.2,", "1,root.sg1.d2,11.1,", "2,root.sg1.d2,22.2,", + "1,root.sg1.d1,false,", + "2,root.sg1.d1,false,", + "1,root.sg1.d2,false,", + "2,root.sg1.d2,false,", }; resultSetEqualTest( "SELECT * FROM root.sg1.d1,root.sg1.d2,root.sg1.d6 soffset 1 slimit 1 ALIGN BY DEVICE;", @@ -957,7 +960,10 @@ public void sLimitOffsetTest() { retArray); retArray = new String[] { - "1,root.sg2.d1,1.1,", "2,root.sg2.d1,2.2,", "1,root.sg2.d2,11.1,", "2,root.sg2.d2,22.2,", + "1,root.sg2.d1,false,", + "2,root.sg2.d1,false,", + "1,root.sg2.d2,false,", + "2,root.sg2.d2,false,", }; resultSetEqualTest( "SELECT * FROM root.sg2.d1,root.sg2.d2,root.sg2.d6 soffset 1 slimit 1 ALIGN BY DEVICE;", @@ -967,7 +973,7 @@ public void sLimitOffsetTest() { @Test public void emptyResultTest() { - String[] expectedHeader = new String[] {"Time,Device,s3,s1,s2"}; + String[] expectedHeader = new String[] {"Time,Device,s1,s2,s3"}; String[] retArray = new String[] {}; // We set timePartitionInterval to 1 in IoTDBAlignByDeviceWithTemplate3IT, the where predicate // is changed from 'time>=now()-1d' to 'time>=now()-1ms' to reduce memory use because of the diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/auth/IoTDBAuthIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/auth/IoTDBAuthIT.java index 1d451edf9730b..81742676a136f 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/auth/IoTDBAuthIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/auth/IoTDBAuthIT.java @@ -363,8 +363,8 @@ public void templateQueryTest() throws SQLException { ResultSet set1 = adminStmt.executeQuery("SELECT * from root.sg.aligned_template"); assertEquals(3, set1.getMetaData().getColumnCount()); - assertEquals("root.sg.aligned_template.temperature", set1.getMetaData().getColumnName(2)); - assertEquals("root.sg.aligned_template.status", set1.getMetaData().getColumnName(3)); + assertEquals("root.sg.aligned_template.status", set1.getMetaData().getColumnName(2)); + assertEquals("root.sg.aligned_template.temperature", set1.getMetaData().getColumnName(3)); ResultSet set2 = userStmt.executeQuery("SELECT * from root.sg.aligned_template"); assertEquals(2, set2.getMetaData().getColumnCount()); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/schematree/visitor/SchemaTreeVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/schematree/visitor/SchemaTreeVisitor.java index d05d1a1ff2a88..ab2f2ec6e2fe5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/schematree/visitor/SchemaTreeVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/schematree/visitor/SchemaTreeVisitor.java @@ -29,16 +29,20 @@ import org.apache.tsfile.write.schema.IMeasurementSchema; import java.util.ArrayList; +import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.NoSuchElementException; +import java.util.TreeMap; import static org.apache.iotdb.commons.schema.SchemaConstant.NON_TEMPLATE; public abstract class SchemaTreeVisitor extends AbstractTreeVisitor { private Map templateMap; + // Reuse the ordered snapshot for devices that share a template. + private final Map> templateMeasurements = new HashMap<>(); protected SchemaTreeVisitor() {} @@ -63,6 +67,7 @@ public List getAllResult() { public void setTemplateMap(Map templateMap) { this.templateMap = templateMap; + templateMeasurements.clear(); } @Override @@ -123,10 +128,12 @@ protected Iterator getChildrenIterator(SchemaNode parent) { && parent.getAsEntityNode().getTemplateId() != NON_TEMPLATE) { return new Iterator() { private final Iterator templateChildrenIterator = - templateMap - .get(parent.getAsEntityNode().getTemplateId()) - .getSchemaMap() - .values() + templateMeasurements + .computeIfAbsent( + parent.getAsEntityNode().getTemplateId(), + templateId -> + new ArrayList<>( + new TreeMap<>(templateMap.get(templateId).getSchemaMap()).values())) .iterator(); private final Iterator directChildrenIterator = parent.getChildrenIterator(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/schematree/visitor/SchemaTreeVisitorWithLimitOffsetWrapper.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/schematree/visitor/SchemaTreeVisitorWithLimitOffsetWrapper.java index 208b63a62ae40..5d7382e79570e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/schematree/visitor/SchemaTreeVisitorWithLimitOffsetWrapper.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/schematree/visitor/SchemaTreeVisitorWithLimitOffsetWrapper.java @@ -42,13 +42,6 @@ public SchemaTreeVisitorWithLimitOffsetWrapper( this.limit = limit; this.offset = offset; hasLimit = limit > 0 || offset > 0; - - if (hasLimit) { - while (curOffset < offset && visitor.hasNext()) { - visitor.next(); - curOffset++; - } - } } public void setTemplateMap(Map templateMap) { @@ -57,11 +50,12 @@ public void setTemplateMap(Map templateMap) { @Override public boolean hasNext() { - if (hasLimit) { - return count < limit && visitor.hasNext(); - } else { - return visitor.hasNext(); + // Template metadata is set after construction, so defer traversal until iteration starts. + while (curOffset < offset && visitor.hasNext()) { + visitor.next(); + curOffset++; } + return (limit == 0 || count < limit) && visitor.hasNext(); } @Override @@ -119,12 +113,6 @@ public void reset() { visitor.reset(); count = 0; curOffset = 0; - if (hasLimit) { - while (curOffset < offset && visitor.hasNext()) { - visitor.next(); - curOffset++; - } - } } public int getNextOffset() { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java index 018f171b13252..a81a8bcc767b2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java @@ -1214,7 +1214,7 @@ private void analyzeGroupByTag( List tagKeys = queryStatement.getGroupByTagComponent().getTagKeys(); Map, LinkedHashMap>> - tagValuesToGroupedTimeseriesOperands = new HashMap<>(); + tagValuesToGroupedTimeseriesOperands = new LinkedHashMap<>(); LinkedHashMap> outputExpressionToRawExpressionsMap = new LinkedHashMap<>(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/TemplatedAggregationAnalyze.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/TemplatedAggregationAnalyze.java index 0de197964e245..ffec2dd67bef4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/TemplatedAggregationAnalyze.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/TemplatedAggregationAnalyze.java @@ -45,6 +45,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.TreeMap; import static org.apache.iotdb.calc.utils.constant.SqlConstant.COUNT_TIME; import static org.apache.iotdb.commons.schema.column.ColumnHeaderConstant.ENDTIME; @@ -106,6 +107,11 @@ static boolean canBuildAggregationPlanUseTemplate( return false; } + if (outputExpressions.isEmpty()) { + analysis.setFinishQueryAfterAnalyze(true); + return true; + } + analyzeDeviceToWhere(analysis, queryStatement); if (deviceList.isEmpty()) { analysis.setFinishQueryAfterAnalyze(true); @@ -180,6 +186,13 @@ private static boolean analyzeSelect( if (selectExpression instanceof FunctionExpression && COUNT_TIME.equalsIgnoreCase( ((FunctionExpression) selectExpression).getFunctionName())) { + if (paginationController.hasCurOffset()) { + paginationController.consumeOffset(); + continue; + } + if (!paginationController.hasCurLimit()) { + break; + } FunctionExpression countTimeExpression = (FunctionExpression) selectExpression; // The parsed count_time(*) expression belongs to the statement and may be analyzed again // after a dispatch failure. Build a template-only expression instead of replacing its @@ -203,6 +216,7 @@ private static boolean analyzeSelect( analysis .getExpressionTypes() .put(NodeRef.of(templatedCountTimeExpression), TSDataType.INT64); + paginationController.consumeLimit(); continue; } @@ -214,7 +228,9 @@ private static boolean analyzeSelect( && "*".equalsIgnoreCase(selectExpression.getExpressions().get(0).getOutputSymbol())) { subExpressions = new ArrayList<>(); FunctionExpression functionExpression = (FunctionExpression) selectExpression; - for (Map.Entry entry : template.getSchemaMap().entrySet()) { + // Use the same stable wildcard order as raw-data queries, before applying pagination. + for (Map.Entry entry : + new TreeMap<>(template.getSchemaMap()).entrySet()) { FunctionExpression subFunctionExpression = new FunctionExpression( functionExpression.getFunctionName(), @@ -241,6 +257,7 @@ private static boolean analyzeSelect( selectExpressions.add(expression); aggregationExpressions.add(expression); analyzeExpressionType(analysis, expression); + paginationController.consumeLimit(); } else { break; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/TemplatedAnalyze.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/TemplatedAnalyze.java index 350cc5d090ea5..287b2e6460142 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/TemplatedAnalyze.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/TemplatedAnalyze.java @@ -56,6 +56,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.TreeMap; import java.util.stream.Collectors; import static org.apache.iotdb.db.queryengine.metric.QueryPlanCostMetricSet.PARTITION_FETCHER; @@ -134,7 +135,9 @@ public static boolean canBuildPlanUseTemplate( for (ResultColumn resultColumn : queryStatement.getSelectComponent().getResultColumns()) { Expression expression = resultColumn.getExpression(); if ("*".equals(expression.getOutputSymbol())) { - for (Map.Entry entry : template.getSchemaMap().entrySet()) { + // Template map iteration can change after serialization. Order columns before pagination. + for (Map.Entry entry : + new TreeMap<>(template.getSchemaMap()).entrySet()) { if (paginationController.hasCurOffset()) { paginationController.consumeOffset(); } else if (paginationController.hasCurLimit()) { @@ -168,6 +171,7 @@ public static boolean canBuildPlanUseTemplate( // reserve memory for this expression context.reserveMemoryForFrontEnd(measurementPath.ramBytesUsed()); outputExpressions.add(new Pair<>(measurementPath, resultColumn.getAlias())); + paginationController.consumeLimit(); } else { break; } @@ -183,6 +187,11 @@ public static boolean canBuildPlanUseTemplate( analyzeSelect(queryStatement, analysis, outputExpressions, template); + if (outputExpressions.isEmpty()) { + analysis.setFinishQueryAfterAnalyze(true); + return true; + } + analyzeDeviceToWhere(analysis, queryStatement); if (analysis.getWhereExpression() != null && analysis.getWhereExpression().equals(ConstantOperand.FALSE)) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanBuilder.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanBuilder.java index b976ce4ba2c66..e3b7566cb5754 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanBuilder.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanBuilder.java @@ -118,7 +118,6 @@ import java.util.Arrays; import java.util.Collection; import java.util.Collections; -import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashMap; import java.util.LinkedHashSet; @@ -820,7 +819,7 @@ private PlanNode createGroupByTagNode( GroupByTimeParameter groupByTimeParameter, Ordering scanOrder) { Map, List> tagValuesToAggregationDescriptors = - new HashMap<>(); + new LinkedHashMap<>(); for (List tagValues : tagValuesToGroupedTimeseriesOperands.keySet()) { LinkedHashMap> groupedTimeseriesOperands = tagValuesToGroupedTimeseriesOperands.get(tagValues); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/process/GroupByTagNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/process/GroupByTagNode.java index d96100bfac81a..b5485738f536f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/process/GroupByTagNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/process/GroupByTagNode.java @@ -38,7 +38,7 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.util.ArrayList; -import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Map.Entry; @@ -253,7 +253,7 @@ public static GroupByTagNode deserialize(ByteBuffer byteBuffer) { // Tag values to aggregation descriptors. int numOfEntries = ReadWriteIOUtils.readInt(byteBuffer); Map, List> tagValuesToAggregationDescriptors = - new HashMap<>(); + new LinkedHashMap<>(); while (numOfEntries > 0) { List tagValues = ReadWriteIOUtils.readStringList(byteBuffer); List aggregationDescriptors = new ArrayList<>(); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/GroupByTagPlanningOrderTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/GroupByTagPlanningOrderTest.java new file mode 100644 index 0000000000000..0f2140403cb67 --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/GroupByTagPlanningOrderTest.java @@ -0,0 +1,124 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.analyze; + +import org.apache.iotdb.common.rpc.thrift.TEndPoint; +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.path.PathPatternTree; +import org.apache.iotdb.commons.queryengine.common.SessionInfo; +import org.apache.iotdb.db.queryengine.common.MPPQueryContext; +import org.apache.iotdb.db.queryengine.common.QueryId; +import org.apache.iotdb.db.queryengine.common.schematree.ClusterSchemaTree; +import org.apache.iotdb.db.queryengine.plan.expression.Expression; +import org.apache.iotdb.db.queryengine.plan.parser.StatementGenerator; +import org.apache.iotdb.db.queryengine.plan.planner.LogicalPlanBuilder; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.process.GroupByTagNode; +import org.apache.iotdb.db.queryengine.plan.statement.component.Ordering; + +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.Test; + +import java.time.ZoneId; +import java.time.ZonedDateTime; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.assertEquals; + +public class GroupByTagPlanningOrderTest { + + @Test + public void testAnalysisPreservesGroupEncounterOrder() throws Exception { + Analysis analysis = analyze(createQueryContext()); + assertEquals( + Arrays.asList(Collections.singletonList("v1"), Collections.singletonList("v0")), + new ArrayList<>(analysis.getTagValuesToGroupedTimeseriesOperands().keySet())); + } + + @Test + public void testLogicalPlanPreservesAnalyzedGroupOrder() throws Exception { + MPPQueryContext context = createQueryContext(); + Analysis analysis = analyze(context); + Map, LinkedHashMap>> groups = new LinkedHashMap<>(); + for (String value : Arrays.asList("v1", "v0")) { + List tagValues = Collections.singletonList(value); + groups.put(tagValues, analysis.getTagValuesToGroupedTimeseriesOperands().get(tagValues)); + } + + GroupByTagNode node = + (GroupByTagNode) + new LogicalPlanBuilder(analysis, context) + .planRawDataSource(analysis.getSourceExpressions(), Ordering.ASC, 0, 0, false) + .planGroupByTag( + analysis.getCrossGroupByExpressions(), + analysis.getTagKeys(), + groups, + null, + Ordering.ASC) + .getRoot(); + assertEquals( + new ArrayList<>(groups.keySet()), + new ArrayList<>(node.getTagValuesToAggregationDescriptors().keySet())); + } + + private Analysis analyze(MPPQueryContext context) throws Exception { + ClusterSchemaTree testSchemaTree = new ClusterSchemaTree(); + testSchemaTree.setDatabases(Collections.singleton("root.sg")); + for (int i = 1; i <= 2; i++) { + testSchemaTree.appendSingleMeasurement( + new PartialPath("root.sg.d" + i + ".s1"), + new MeasurementSchema("s1", TSDataType.INT32), + Collections.singletonMap("key1", i == 1 ? "v1" : "v0"), + null, + null, + false); + } + FakeSchemaFetcherImpl schemaFetcher = + new FakeSchemaFetcherImpl() { + @Override + public ClusterSchemaTree fetchSchemaWithTags( + PathPatternTree patternTree, + boolean withTemplate, + MPPQueryContext queryContext, + boolean canSeeAuditDB) { + return testSchemaTree; + } + }; + return new Analyzer(context, new FakePartitionFetcherImpl(), schemaFetcher) + .analyze( + StatementGenerator.createStatement( + "select count(s1) from root.sg.* group by tags(key1)", + ZonedDateTime.now().getOffset())); + } + + private MPPQueryContext createQueryContext() { + return new MPPQueryContext( + "", + new QueryId("test_query"), + new SessionInfo(0, "root", ZoneId.systemDefault()), + new TEndPoint(), + new TEndPoint()); + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/TemplateQueryOrderTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/TemplateQueryOrderTest.java new file mode 100644 index 0000000000000..6e8ef9f019e37 --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/TemplateQueryOrderTest.java @@ -0,0 +1,255 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.analyze; + +import org.apache.iotdb.common.rpc.thrift.TEndPoint; +import org.apache.iotdb.commons.path.MeasurementPath; +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.queryengine.common.SessionInfo; +import org.apache.iotdb.commons.schema.template.Template; +import org.apache.iotdb.db.queryengine.common.MPPQueryContext; +import org.apache.iotdb.db.queryengine.common.QueryId; +import org.apache.iotdb.db.queryengine.common.schematree.ClusterSchemaTree; +import org.apache.iotdb.db.queryengine.plan.parser.StatementGenerator; +import org.apache.iotdb.db.queryengine.plan.statement.crud.QueryStatement; + +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.junit.Test; + +import java.time.ZoneId; +import java.time.ZonedDateTime; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class TemplateQueryOrderTest { + + @Test + public void testRawWildcardOrderAfterSerialization() throws Exception { + assertQueryOrder("*", false); + } + + @Test + public void testAggregationWildcardOrderAfterSerialization() throws Exception { + assertQueryOrder("count(*)", true); + } + + @Test + public void testSchemaTreeWildcardOrderAfterSerialization() throws Exception { + for (Template template : createTemplateVersions()) { + List expected = sortedMeasurements(template); + ClusterSchemaTree schemaTree = createSchemaTree(template); + PartialPath pattern = new PartialPath("root.sg.d1.*"); + assertEquals( + expected, + schemaTree.searchMeasurementPaths(pattern).left.stream() + .map(MeasurementPath::getMeasurement) + .collect(Collectors.toList())); + assertEquals( + expected.subList(3, 8), + schemaTree.searchMeasurementPaths(pattern, 5, 3, false).left.stream() + .map(MeasurementPath::getMeasurement) + .collect(Collectors.toList())); + } + } + + @Test + public void testExplicitColumnsHonorSeriesLimit() throws Exception { + for (Template template : createTemplateVersions()) { + List measurements = sortedMeasurements(template); + assertEquals( + measurements.subList(3, 8), + analyzeOutputSymbols( + template, + "select " + + String.join(", ", measurements) + + " from root.sg.* slimit 5 soffset 3 align by device")); + } + } + + @Test + public void testExplicitAggregationsHonorSeriesLimit() throws Exception { + for (Template template : createTemplateVersions()) { + List aggregations = + sortedMeasurements(template).stream() + .map(name -> "count(" + name + ")") + .collect(Collectors.toList()); + assertEquals( + aggregations.subList(3, 8), + analyzeOutputSymbols( + template, + "select " + + String.join(", ", aggregations) + + " from root.sg.* slimit 5 soffset 3 align by device")); + } + } + + @Test + public void testSchemaTreeOffsetWithoutLimit() throws Exception { + for (Template template : createTemplateVersions()) { + List expected = sortedMeasurements(template); + assertEquals( + expected.subList(3, expected.size()), + createSchemaTree(template) + .searchMeasurementPaths(new PartialPath("root.sg.d1.*"), 0, 3, false) + .left + .stream() + .map(MeasurementPath::getMeasurement) + .collect(Collectors.toList())); + } + } + + @Test + public void testCountTimeHonorsSeriesOffset() throws Exception { + for (Template template : createTemplateVersions()) { + assertEmptyProjectionFinishesQuery( + template, "select count_time(*) from root.sg.* slimit 1 soffset 1 align by device"); + } + } + + @Test + public void testRawSeriesOffsetBeyondMeasurementsFinishesQuery() throws Exception { + for (Template template : createTemplateVersions()) { + assertEmptyProjectionFinishesQuery( + template, "select * from root.sg.* slimit 1 soffset 16 align by device"); + } + } + + @Test + public void testAggregationSeriesOffsetBeyondMeasurementsFinishesQuery() throws Exception { + for (Template template : createTemplateVersions()) { + assertEmptyProjectionFinishesQuery( + template, "select count(*) from root.sg.* slimit 1 soffset 16 align by device"); + } + } + + private void assertEmptyProjectionFinishesQuery(Template template, String sql) throws Exception { + Analysis analysis = + analyze( + template, + sql, + Arrays.asList(new PartialPath("root.sg.d1"), new PartialPath("root.sg.d2"))); + assertTrue(analysis.getOutputExpressions().isEmpty()); + assertTrue(analysis.isFinishQueryAfterAnalyze()); + } + + private void assertQueryOrder(String projection, boolean aggregation) throws Exception { + for (Template template : createTemplateVersions()) { + List expected = sortedMeasurements(template); + if (aggregation) { + expected = + expected.stream().map(name -> "count(" + name + ")").collect(Collectors.toList()); + } + assertEquals( + expected, + analyzeOutputSymbols( + template, "select " + projection + " from root.sg.* align by device")); + assertEquals( + expected.subList(3, 8), + analyzeOutputSymbols( + template, + "select " + projection + " from root.sg.* slimit 5 soffset 3 align by device")); + } + } + + private List analyzeOutputSymbols(Template template, String sql) throws Exception { + return analyze(template, sql, Collections.emptyList()).getOutputExpressions().stream() + .map(pair -> pair.left.getOutputSymbol()) + .collect(Collectors.toList()); + } + + private Analysis analyze(Template template, String sql, List devices) + throws Exception { + QueryStatement statement = + (QueryStatement) StatementGenerator.createStatement(sql, ZonedDateTime.now().getOffset()); + statement.semanticCheck(); + Analysis analysis = new Analysis(); + MPPQueryContext context = + new MPPQueryContext( + sql, + new QueryId("test_query"), + new SessionInfo(0, "test", ZoneId.systemDefault()), + new TEndPoint(), + new TEndPoint()); + assertTrue( + TemplatedAnalyze.canBuildPlanUseTemplate( + analysis, + statement, + new FakePartitionFetcherImpl(), + createSchemaTree(template), + context, + devices, + new TreeAnalysisMutationJournal())); + return analysis; + } + + private ClusterSchemaTree createSchemaTree(Template template) throws Exception { + ClusterSchemaTree schemaTree = new ClusterSchemaTree(); + schemaTree.setDatabases(Collections.singleton("root.sg")); + for (String device : Arrays.asList("root.sg.d1", "root.sg.d2")) { + schemaTree.appendTemplateDevice(new PartialPath(device), true, template.getId(), template); + } + return schemaTree; + } + + private List sortedMeasurements(Template template) { + List measurements = new ArrayList<>(template.getSchemaMap().keySet()); + Collections.sort(measurements); + return measurements; + } + + private List