diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBDatabaseIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBDatabaseIT.java index 4736d9b0521d..c947a4979888 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBDatabaseIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBDatabaseIT.java @@ -109,6 +109,7 @@ public void testManageDatabase() { statement.execute("alter database if exists test1 set properties ttl='INF'"); statement.execute("alter database test set properties ttl=default"); + statement.execute("alter database test set properties need_last_cache=false"); String[] databaseNames = new String[] {"test"}; String[] TTLs = new String[] {"INF"}; @@ -160,6 +161,7 @@ public void testManageDatabase() { assertEquals(timePartitionInterval[cnt], resultSet.getLong(5)); assertEquals(schemaRegionGroupNum[cnt], resultSet.getInt(6)); assertEquals(dataRegionGroupNum[cnt], resultSet.getInt(7)); + assertFalse(resultSet.getBoolean(8)); cnt++; } assertEquals(databaseNames.length, cnt); @@ -426,7 +428,8 @@ public void testInformationSchema() throws SQLException { "data_replication_factor,INT32,ATTRIBUTE,", "time_partition_interval,INT64,ATTRIBUTE,", "schema_region_group_num,INT32,ATTRIBUTE,", - "data_region_group_num,INT32,ATTRIBUTE,"))); + "data_region_group_num,INT32,ATTRIBUTE,", + "need_last_cache,BOOLEAN,ATTRIBUTE,"))); TestUtils.assertResultSetEqual( statement.executeQuery("desc tables"), "ColumnName,DataType,Category,", @@ -437,7 +440,8 @@ public void testInformationSchema() throws SQLException { "ttl(ms),STRING,ATTRIBUTE,", "status,STRING,ATTRIBUTE,", "comment,STRING,ATTRIBUTE,", - "table_type,STRING,ATTRIBUTE,"))); + "table_type,STRING,ATTRIBUTE,", + "need_last_cache,BOOLEAN,ATTRIBUTE,"))); TestUtils.assertResultSetEqual( statement.executeQuery("desc columns"), "ColumnName,DataType,Category,", @@ -605,41 +609,41 @@ public void testInformationSchema() throws SQLException { statement.execute( "create table test.test (a tag, b attribute, c int32 comment 'turbine') comment 'test'"); statement.execute( - "CREATE VIEW test.view_table (tag1 STRING TAG,tag2 STRING TAG,s11 INT32 FIELD,s3 INT32 FIELD FROM s2) RESTRICT WITH (ttl=100) AS root.\"a\".**"); + "CREATE VIEW test.view_table (tag1 STRING TAG,tag2 STRING TAG,s11 INT32 FIELD,s3 INT32 FIELD FROM s2) RESTRICT WITH (ttl=100, need_last_cache=true) AS root.\"a\".**"); TestUtils.assertResultSetEqual( statement.executeQuery("select * from databases"), - "database,ttl(ms),schema_replication_factor,data_replication_factor,time_partition_interval,schema_region_group_num,data_region_group_num,", + "database,ttl(ms),schema_replication_factor,data_replication_factor,time_partition_interval,schema_region_group_num,data_region_group_num,need_last_cache,", new HashSet<>( Arrays.asList( - "information_schema,INF,null,null,null,null,null,", - "test,INF,1,1,604800000,0,0,"))); + "information_schema,INF,null,null,null,null,null,false,", + "test,INF,1,1,604800000,0,0,true,"))); TestUtils.assertResultSetEqual( statement.executeQuery("show devices from tables where status = 'USING'"), - "database,table_name,ttl(ms),status,comment,table_type,", + "database,table_name,ttl(ms),status,comment,table_type,need_last_cache,", new HashSet<>( Arrays.asList( - "information_schema,databases,INF,USING,null,SYSTEM VIEW,", - "information_schema,tables,INF,USING,null,SYSTEM VIEW,", - "information_schema,columns,INF,USING,null,SYSTEM VIEW,", - "information_schema,queries,INF,USING,null,SYSTEM VIEW,", - "information_schema,regions,INF,USING,null,SYSTEM VIEW,", - "information_schema,topics,INF,USING,null,SYSTEM VIEW,", - "information_schema,pipe_plugins,INF,USING,null,SYSTEM VIEW,", - "information_schema,pipes,INF,USING,null,SYSTEM VIEW,", - "information_schema,subscriptions,INF,USING,null,SYSTEM VIEW,", - "information_schema,views,INF,USING,null,SYSTEM VIEW,", - "information_schema,functions,INF,USING,null,SYSTEM VIEW,", - "information_schema,configurations,INF,USING,null,SYSTEM VIEW,", - "information_schema,keywords,INF,USING,null,SYSTEM VIEW,", - "information_schema,nodes,INF,USING,null,SYSTEM VIEW,", - "information_schema,config_nodes,INF,USING,null,SYSTEM VIEW,", - "information_schema,data_nodes,INF,USING,null,SYSTEM VIEW,", + "information_schema,databases,INF,USING,null,SYSTEM VIEW,false,", + "information_schema,tables,INF,USING,null,SYSTEM VIEW,false,", + "information_schema,columns,INF,USING,null,SYSTEM VIEW,false,", + "information_schema,queries,INF,USING,null,SYSTEM VIEW,false,", + "information_schema,regions,INF,USING,null,SYSTEM VIEW,false,", + "information_schema,topics,INF,USING,null,SYSTEM VIEW,false,", + "information_schema,pipe_plugins,INF,USING,null,SYSTEM VIEW,false,", + "information_schema,pipes,INF,USING,null,SYSTEM VIEW,false,", + "information_schema,subscriptions,INF,USING,null,SYSTEM VIEW,false,", + "information_schema,views,INF,USING,null,SYSTEM VIEW,false,", + "information_schema,functions,INF,USING,null,SYSTEM VIEW,false,", + "information_schema,configurations,INF,USING,null,SYSTEM VIEW,false,", + "information_schema,keywords,INF,USING,null,SYSTEM VIEW,false,", + "information_schema,nodes,INF,USING,null,SYSTEM VIEW,false,", + "information_schema,config_nodes,INF,USING,null,SYSTEM VIEW,false,", + "information_schema,data_nodes,INF,USING,null,SYSTEM VIEW,false,", "information_schema,connections,INF,USING,null,SYSTEM VIEW,", - "information_schema,current_queries,INF,USING,null,SYSTEM VIEW,", - "information_schema,queries_costs_histogram,INF,USING,null,SYSTEM VIEW,", - "test,test,INF,USING,test,BASE TABLE,", - "test,view_table,100,USING,null,VIEW FROM TREE,"))); + "information_schema,current_queries,INF,USING,null,SYSTEM VIEW,false,", + "information_schema,queries_costs_histogram,INF,USING,null,SYSTEM VIEW,false,", + "test,test,INF,USING,test,BASE TABLE,true,", + "test,view_table,100,USING,null,VIEW FROM TREE,true,"))); TestUtils.assertResultSetEqual( statement.executeQuery("count devices from tables where status = 'USING'"), "count(devices),", @@ -690,7 +694,7 @@ public void testInformationSchema() throws SQLException { statement.executeQuery("select * from views"), "database,table_name,view_definition,", Collections.singleton( - "test,view_table,CREATE VIEW \"view_table\" (\"tag1\" STRING TAG,\"tag2\" STRING TAG,\"s11\" INT32 FIELD,\"s3\" INT32 FIELD FROM \"s2\") RESTRICT WITH (ttl=100) AS root.\"a\".**,")); + "test,view_table,CREATE VIEW \"view_table\" (\"tag1\" STRING TAG,\"tag2\" STRING TAG,\"s11\" INT32 FIELD,\"s3\" INT32 FIELD FROM \"s2\") RESTRICT WITH (ttl=100, need_last_cache=true) AS root.\"a\".**,")); TestUtils.assertResultSetEqual( statement.executeQuery( @@ -818,8 +822,8 @@ public void testDBAuth() throws SQLException { Collections.singleton("information_schema,INF,null,null,null,")); TestUtils.assertResultSetEqual( userStmt.executeQuery("select * from information_schema.databases"), - "database,ttl(ms),schema_replication_factor,data_replication_factor,time_partition_interval,schema_region_group_num,data_region_group_num,", - Collections.singleton("information_schema,INF,null,null,null,null,null,")); + "database,ttl(ms),schema_replication_factor,data_replication_factor,time_partition_interval,schema_region_group_num,data_region_group_num,need_last_cache,", + Collections.singleton("information_schema,INF,null,null,null,null,null,false,")); } try (final Connection adminCon = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBTableIT.java index 7ad5172fca41..9e4f7623cbb0 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBTableIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBTableIT.java @@ -130,6 +130,7 @@ public void testManageTable() { String[] ttls = new String[] {"INF"}; String[] statuses = new String[] {"USING"}; String[] comments = new String[] {"test"}; + String[] needLastCaches = new String[] {"true"}; statement.execute("use test2"); @@ -149,6 +150,7 @@ public void testManageTable() { assertEquals(ttls[cnt], resultSet.getString(2)); assertEquals(statuses[cnt], resultSet.getString(3)); assertEquals(comments[cnt], resultSet.getString(4)); + assertEquals(needLastCaches[cnt], resultSet.getString(6)); cnt++; } assertEquals(tableNames.length, cnt); @@ -193,6 +195,9 @@ public void testManageTable() { statement.execute("comment on table test1.table1 is 'new_test'"); comments = new String[] {"new_test"}; + + statement.execute("alter table test1.table1 set properties need_last_cache=false"); + needLastCaches = new String[] {"false"}; // using SHOW tables from try (final ResultSet resultSet = statement.executeQuery("SHOW tables details from test1")) { int cnt = 0; @@ -206,6 +211,7 @@ public void testManageTable() { assertEquals(tableNames[cnt], resultSet.getString(1)); assertEquals(ttls[cnt], resultSet.getString(2)); assertEquals(comments[cnt], resultSet.getString(4)); + assertEquals(needLastCaches[cnt], resultSet.getString(6)); cnt++; } assertEquals(tableNames.length, cnt); @@ -309,7 +315,7 @@ public void testManageTable() { statement.executeQuery("show create table table2"), "Table,Create Table,", Collections.singleton( - "table2,CREATE TABLE \"table2\" (\"region_id\" STRING TAG,\"plant_id\" STRING TAG,\"color\" STRING ATTRIBUTE,\"temperature\" FLOAT FIELD,\"speed\" DOUBLE FIELD COMMENT 'fast') WITH (ttl=6600000),")); + "table2,CREATE TABLE \"table2\" (\"region_id\" STRING TAG,\"plant_id\" STRING TAG,\"color\" STRING ATTRIBUTE,\"temperature\" FLOAT FIELD,\"speed\" DOUBLE FIELD COMMENT 'fast') WITH (ttl=6600000, need_last_cache=true),")); try { statement.execute("alter table table2 add column speed DOUBLE FIELD"); @@ -660,7 +666,7 @@ public void testTableAuth() throws SQLException { Assert.assertThrows(SQLException.class, () -> userStmt.execute("select * from db.test")); TestUtils.assertResultSetEqual( userStmt.executeQuery("select * from information_schema.tables where database = 'db'"), - "database,table_name,ttl(ms),status,comment,table_type,", + "database,table_name,ttl(ms),status,comment,table_type,need_last_cache,", Collections.emptySet()); TestUtils.assertResultSetEqual( userStmt.executeQuery("select * from information_schema.columns where database = 'db'"), @@ -1073,8 +1079,8 @@ public void testTreeViewTable() throws Exception { TestUtils.assertResultSetEqual( statement.executeQuery("show tables details"), - "TableName,TTL(ms),Status,Comment,TableType,", - Collections.singleton("view_table,100,USING,comment,VIEW FROM TREE,")); + "TableName,TTL(ms),Status,Comment,TableType,NeedLastCache,", + Collections.singleton("view_table,100,USING,comment,VIEW FROM TREE,true,")); TestUtils.assertResultSetEqual( statement.executeQuery("desc view_table"), @@ -1209,14 +1215,14 @@ public void testTreeViewTable() throws Exception { statement.executeQuery("show create view view_table"), "View,Create View,", Collections.singleton( - "view_table,CREATE VIEW \"view_table\" (\"tag1\" STRING TAG,\"tag2\" STRING TAG,\"s11\" INT32 FIELD,\"s3\" STRING FIELD FROM \"s2\") RESTRICT WITH (ttl=100) AS root.\"重庆\".**,")); + "view_table,CREATE VIEW \"view_table\" (\"tag1\" STRING TAG,\"tag2\" STRING TAG,\"s11\" INT32 FIELD,\"s3\" STRING FIELD FROM \"s2\") RESTRICT WITH (ttl=100, need_last_cache=true) AS root.\"重庆\".**,")); // Can also use "show create table" TestUtils.assertResultSetEqual( statement.executeQuery("show create table view_table"), "View,Create View,", Collections.singleton( - "view_table,CREATE VIEW \"view_table\" (\"tag1\" STRING TAG,\"tag2\" STRING TAG,\"s11\" INT32 FIELD,\"s3\" STRING FIELD FROM \"s2\") RESTRICT WITH (ttl=100) AS root.\"重庆\".**,")); + "view_table,CREATE VIEW \"view_table\" (\"tag1\" STRING TAG,\"tag2\" STRING TAG,\"s11\" INT32 FIELD,\"s3\" STRING FIELD FROM \"s2\") RESTRICT WITH (ttl=100, need_last_cache=true) AS root.\"重庆\".**,")); statement.execute("create table a ()"); try { diff --git a/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IdentifierParser.g4 b/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IdentifierParser.g4 index 4a01b352384c..c388226842bf 100644 --- a/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IdentifierParser.g4 +++ b/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IdentifierParser.g4 @@ -153,6 +153,7 @@ keyWords | MODELS | MODIFY | NAN + | NEED_LAST_CACHE | NODEID | NODES | NONE diff --git a/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 b/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 index 357a004336c2..8693488eae9e 100644 --- a/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 +++ b/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 @@ -111,6 +111,7 @@ databaseAttributesClause databaseAttributeClause : databaseAttributeKey operator_eq INTEGER_LITERAL + | NEED_LAST_CACHE operator_eq boolean_literal ; databaseAttributeKey diff --git a/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4 b/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4 index 5696d5da32a4..e8b5c17eed6b 100644 --- a/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4 +++ b/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4 @@ -1194,6 +1194,10 @@ DATA_REGION_GROUP_NUM : D A T A '_' R E G I O N '_' G R O U P '_' N U M ; +NEED_LAST_CACHE + : N E E D '_' L A S T '_' C A C H E + ; + CURRENT_TIMESTAMP : C U R R E N T '_' T I M E S T A M P ; diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/schema/ClusterSchemaManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/schema/ClusterSchemaManager.java index 0648965cda61..d4825d866589 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/schema/ClusterSchemaManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/schema/ClusterSchemaManager.java @@ -382,6 +382,7 @@ public TShowDatabaseResp showDatabase(final GetDatabasePlan getDatabasePlan) { databaseInfo.setDataReplicationFactor(databaseSchema.getDataReplicationFactor()); databaseInfo.setTimePartitionOrigin(databaseSchema.getTimePartitionOrigin()); databaseInfo.setTimePartitionInterval(databaseSchema.getTimePartitionInterval()); + databaseInfo.setNeedLastCache(databaseSchema.isNeedLastCache()); databaseInfo.setMinSchemaRegionNum( getMinRegionGroupNum(database, TConsensusGroupType.SchemaRegion)); databaseInfo.setMaxSchemaRegionNum( @@ -875,6 +876,10 @@ public static TSStatus enrichDatabaseSchemaWithDefaultProperties( "Failed to create database. The timePartitionInterval should be positive."); } + if (!databaseSchema.isSetNeedLastCache()) { + databaseSchema.setNeedLastCache(true); + } + if (isSystemDatabase || isAuditDatabase) { databaseSchema.setMinSchemaRegionGroupNum(1); } else if (!databaseSchema.isSetMinSchemaRegionGroupNum()) { diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ClusterSchemaInfo.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ClusterSchemaInfo.java index 62fe57fb0b7c..1bce51c9bd3d 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ClusterSchemaInfo.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ClusterSchemaInfo.java @@ -136,6 +136,7 @@ import static org.apache.iotdb.commons.schema.SchemaConstant.SYSTEM_DATABASE_PATTERN; import static org.apache.iotdb.commons.schema.table.Audit.TABLE_MODEL_AUDIT_DATABASE; import static org.apache.iotdb.commons.schema.table.Audit.TREE_MODEL_AUDIT_DATABASE; +import static org.apache.iotdb.commons.schema.table.TsTable.NEED_LAST_CACHE_PROPERTY; import static org.apache.iotdb.commons.schema.table.TsTable.TTL_PROPERTY; /** @@ -271,6 +272,21 @@ public TSStatus alterDatabase(final DatabaseSchemaPlan plan) { currentSchema.getTTL()); } + if (alterSchema.isSetNeedLastCache() + && alterSchema.isNeedLastCache() + != (!currentSchema.isSetNeedLastCache() || currentSchema.isNeedLastCache())) { + if (!currentSchema.isIsTableModel()) { + result.setCode(TSStatusCode.SEMANTIC_ERROR.getStatusCode()); + result.setMessage("The tree model database does not support alter need last cache now."); + return result; + } + currentSchema.setNeedLastCache(alterSchema.isNeedLastCache()); + LOGGER.info( + "[SetNeedLastCache] The need last cache flag of Database: {} is adjusted to: {}", + currentSchema.getName(), + currentSchema.isNeedLastCache()); + } + mTree .getDatabaseNodeByDatabasePath(partialPathName) .getAsMNode() @@ -1285,6 +1301,11 @@ public ShowTableResp showTables(final ShowTablePlan plan) { TreeViewSchema.isTreeViewTable(pair.getLeft()) ? TableType.VIEW_FROM_TREE.ordinal() : TableType.BASE_TABLE.ordinal()); + info.setNeedLastCache( + pair.getLeft() + .getPropValue(NEED_LAST_CACHE_PROPERTY) + .map(Boolean::parseBoolean) + .orElse(true)); return info; }) .collect(Collectors.toList()) @@ -1333,6 +1354,11 @@ public ShowTable4InformationSchemaResp showTables4InformationSchema() { TreeViewSchema.isTreeViewTable(pair.getLeft()) ? TableType.VIEW_FROM_TREE.ordinal() : TableType.BASE_TABLE.ordinal()); + info.setNeedLastCache( + pair.getLeft() + .getPropValue(NEED_LAST_CACHE_PROPERTY) + .map(Boolean::parseBoolean) + .orElse(true)); return info; }) .collect(Collectors.toList())))); diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ConfigMTree.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ConfigMTree.java index a4231e5db83b..cbfd5ec90f61 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ConfigMTree.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ConfigMTree.java @@ -945,6 +945,11 @@ public void setTableProperties( && databaseNode.getDatabaseSchema().isSetTTL() && databaseNode.getDatabaseSchema().getTTL() != Long.MAX_VALUE) { table.addProp(k, String.valueOf(databaseNode.getDatabaseSchema().getTTL())); + } else if (k.equals(TsTable.NEED_LAST_CACHE_PROPERTY) + && databaseNode.getDatabaseSchema().isSetNeedLastCache()) { + table.addProp( + TsTable.NEED_LAST_CACHE_PROPERTY, + String.valueOf(databaseNode.getDatabaseSchema().isNeedLastCache())); } else { table.removeProp(k); } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/CreateTableProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/CreateTableProcedure.java index d0fbdb605d12..a1e1383ac5db 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/CreateTableProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/CreateTableProcedure.java @@ -129,6 +129,10 @@ protected void checkTableExistence(final ConfigNodeProcedureEnv env) { && schema.getTTL() != Long.MAX_VALUE) { table.addProp(TsTable.TTL_PROPERTY, String.valueOf(schema.getTTL())); } + if (!table.getPropValue(TsTable.NEED_LAST_CACHE_PROPERTY).isPresent() + && schema.isSetNeedLastCache()) { + table.addProp(TsTable.NEED_LAST_CACHE_PROPERTY, String.valueOf(schema.isNeedLastCache())); + } setNextState(CreateTableState.PRE_CREATE); } } catch (final MetadataException | DatabaseNotExistsException e) { diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/view/CreateTableViewProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/view/CreateTableViewProcedure.java index b7fb458c8bb0..0ba5a916787c 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/view/CreateTableViewProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/view/CreateTableViewProcedure.java @@ -101,6 +101,11 @@ protected void checkTableExistence(final ConfigNodeProcedureEnv env) { && schema.getTTL() != Long.MAX_VALUE) { table.addProp(TsTable.TTL_PROPERTY, String.valueOf(schema.getTTL())); } + if (!table.getPropValue(TsTable.NEED_LAST_CACHE_PROPERTY).isPresent() + && schema.isSetNeedLastCache()) { + table.addProp( + TsTable.NEED_LAST_CACHE_PROPERTY, String.valueOf(schema.isNeedLastCache())); + } setNextState(CreateTableState.PRE_CREATE); } } catch (final MetadataException | DatabaseNotExistsException e) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/InformationSchemaContentSupplierFactory.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/InformationSchemaContentSupplierFactory.java index b8fd3a094db1..8a2e477df119 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/InformationSchemaContentSupplierFactory.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/InformationSchemaContentSupplierFactory.java @@ -280,6 +280,7 @@ protected void constructLine() { columnBuilders[4].writeLong(currentDatabase.getTimePartitionInterval()); columnBuilders[5].writeInt(currentDatabase.getSchemaRegionNum()); columnBuilders[6].writeInt(currentDatabase.getDataRegionNum()); + columnBuilders[7].writeBoolean(currentDatabase.isNeedLastCache()); resultBuilder.declarePosition(); currentDatabase = null; } @@ -334,6 +335,7 @@ private TableSupplier(final List dataTypes, final UserEntity userEnt table.getTableName(), table.getPropValue(TTL_PROPERTY).orElse(TTL_INFINITE)); info.setState(TableNodeStatus.USING.ordinal()); + info.setNeedLastCache(false); return info; }) .collect(Collectors.toList())); @@ -368,6 +370,7 @@ protected void constructLine() { columnBuilders[5].writeBinary( new Binary(TableType.BASE_TABLE.getName(), TSFileConfig.STRING_CHARSET)); } + columnBuilders[6].writeBoolean(currentTable.isNeedLastCache()); resultBuilder.declarePosition(); currentTable = null; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/ClusterPartitionFetcher.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/ClusterPartitionFetcher.java index 2274762341b5..297c52ce168b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/ClusterPartitionFetcher.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/ClusterPartitionFetcher.java @@ -331,6 +331,10 @@ public SchemaPartition getSchemaPartition( return getOrCreateSchemaPartition(database, deviceIDs, false, null); } + public boolean needLastCache(final String database) { + return partitionCache.isNeedLastCache(database); + } + private SchemaPartition getOrCreateSchemaPartition( final String database, final @Nullable List deviceIDs, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/IPartitionFetcher.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/IPartitionFetcher.java index 0549ec396474..99f1b9f99c3b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/IPartitionFetcher.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/IPartitionFetcher.java @@ -26,6 +26,7 @@ import org.apache.iotdb.commons.path.PathPatternTree; import org.apache.iotdb.mpp.rpc.thrift.TRegionRouteReq; +import org.apache.tsfile.annotations.TableModel; import org.apache.tsfile.file.metadata.IDeviceID; import javax.annotation.Nullable; @@ -123,6 +124,7 @@ SchemaPartition getOrCreateSchemaPartition( * *

The device id shall be [table, seg1, ....] */ + @TableModel SchemaPartition getSchemaPartition( final String database, final @Nullable List deviceIDs); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/partition/PartitionCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/partition/PartitionCache.java index 3c76bd08c1d8..cdff73b4deac 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/partition/PartitionCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/partition/PartitionCache.java @@ -65,6 +65,8 @@ import com.github.benmanes.caffeine.cache.Cache; import com.github.benmanes.caffeine.cache.Caffeine; import org.apache.thrift.TException; +import org.apache.tsfile.annotations.TableModel; +import org.apache.tsfile.annotations.TreeModel; import org.apache.tsfile.file.metadata.IDeviceID; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -97,7 +99,7 @@ public class PartitionCache { private final SeriesPartitionExecutor partitionExecutor; /** the cache of database */ - private final Set databaseCache = new HashSet<>(); + private final Map database2NeedLastCacheCache = new HashMap<>(); /** database -> schemaPartitionTable */ private final Cache schemaPartitionCache; @@ -198,7 +200,7 @@ public void put(final IDeviceID device, final String databaseName) { * @return database name, return {@code null} if cache miss */ private String getDatabaseName(final IDeviceID deviceID) { - for (final String database : databaseCache) { + for (final String database : database2NeedLastCacheCache.keySet()) { if (PathUtils.isStartWith(deviceID, database)) { return database; } @@ -215,7 +217,7 @@ private String getDatabaseName(final IDeviceID deviceID) { private boolean containsDatabase(final String database) { databaseCacheLock.readLock().lock(); try { - return databaseCache.contains(database); + return database2NeedLastCacheCache.containsKey(database); } finally { databaseCacheLock.readLock().unlock(); } @@ -244,7 +246,7 @@ private void fetchDatabaseAndUpdateCache( if (databaseSchemaResp.getStatus().getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { // update all database into cache - updateDatabaseCache(databaseSchemaResp.getDatabaseSchemaMap().keySet()); + updateDatabaseCache(databaseSchemaResp.getDatabaseSchemaMap()); getDatabaseMap(result, deviceIDs, true); } } @@ -253,19 +255,39 @@ private void fetchDatabaseAndUpdateCache( } } + @TreeModel + public boolean isNeedLastCache(final String database) { + Boolean needLastCache = database2NeedLastCacheCache.get(database); + if (Objects.nonNull(needLastCache)) { + return needLastCache; + } + try { + fetchDatabaseAndUpdateCache(false); + } catch (final TException | ClientManagerException e) { + logger.warn( + "Failed to get need_last_cache info for database {}, will put cache anyway, exception: {}", + database, + e.getMessage()); + return true; + } + needLastCache = database2NeedLastCacheCache.get(database); + return Objects.isNull(needLastCache) || needLastCache; + } + /** get all database from configNode and update database cache. */ - private void fetchDatabaseAndUpdateCache() throws ClientManagerException, TException { + private void fetchDatabaseAndUpdateCache(final boolean isTableModel) + throws ClientManagerException, TException { databaseCacheLock.writeLock().lock(); try (final ConfigNodeClient client = configNodeClientManager.borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) { final TGetDatabaseReq req = new TGetDatabaseReq(ROOT_PATH, SchemaConstant.ALL_MATCH_SCOPE_BINARY) - .setIsTableModel(true) + .setIsTableModel(isTableModel) .setCanSeeAuditDB(true); final TDatabaseSchemaResp databaseSchemaResp = client.getMatchedDatabaseSchemas(req); if (databaseSchemaResp.getStatus().getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { // update all database into cache - updateDatabaseCache(databaseSchemaResp.getDatabaseSchemaMap().keySet()); + updateDatabaseCache(databaseSchemaResp.getDatabaseSchemaMap()); } } finally { databaseCacheLock.writeLock().unlock(); @@ -510,13 +532,14 @@ private void getDatabaseCacheResult( } } + @TableModel public void checkAndAutoCreateDatabase( final String database, final boolean isAutoCreate, final String userName) { boolean isExisted = containsDatabase(database); if (!isExisted) { try { // try to fetch database from config node when miss - fetchDatabaseAndUpdateCache(); + fetchDatabaseAndUpdateCache(true); isExisted = containsDatabase(database); if (!isExisted && isAutoCreate) { // try to auto create database of failed device @@ -532,12 +555,28 @@ public void checkAndAutoCreateDatabase( /** * update database cache * - * @param databaseNames the database names that need to update + * @param databases the database names + */ + public void updateDatabaseCache(final Set databases) { + databaseCacheLock.writeLock().lock(); + try { + databases.forEach(database -> database2NeedLastCacheCache.put(database, true)); + } finally { + databaseCacheLock.writeLock().unlock(); + } + } + + /** + * update database cache + * + * @param databaseMap the database names and need last cache that need to update */ - public void updateDatabaseCache(final Set databaseNames) { + public void updateDatabaseCache(final Map databaseMap) { databaseCacheLock.writeLock().lock(); try { - databaseCache.addAll(databaseNames); + databaseMap.forEach( + (database, schema) -> + database2NeedLastCacheCache.put(database, schema.isNeedLastCache())); } finally { databaseCacheLock.writeLock().unlock(); } @@ -547,7 +586,7 @@ public void updateDatabaseCache(final Set databaseNames) { public void removeFromDatabaseCache() { databaseCacheLock.writeLock().lock(); try { - databaseCache.clear(); + database2NeedLastCacheCache.clear(); } finally { databaseCacheLock.writeLock().unlock(); } @@ -1072,7 +1111,7 @@ public void invalidAllCache() { public String toString() { return "PartitionCache{" + ", databaseCache=" - + databaseCache + + database2NeedLastCacheCache + ", replicaSetCache=" + groupIdToReplicaSetMap + ", schemaPartitionCache=" diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java index fc973548ae89..7bae295b3c17 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java @@ -139,6 +139,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.AlterDB; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.AlterPipe; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.AstVisitor; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.BooleanLiteral; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ClearCache; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ColumnDefinition; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.CreateDB; @@ -266,9 +267,11 @@ import static org.apache.iotdb.commons.conf.IoTDBConstant.TTL_INFINITE; import static org.apache.iotdb.commons.executable.ExecutableManager.getUnTrustedUriErrorMsg; import static org.apache.iotdb.commons.executable.ExecutableManager.isUriTrusted; +import static org.apache.iotdb.commons.schema.table.TsTable.NEED_LAST_CACHE_PROPERTY; import static org.apache.iotdb.commons.schema.table.TsTable.TABLE_ALLOWED_PROPERTIES; import static org.apache.iotdb.commons.schema.table.TsTable.TIME_COLUMN_NAME; import static org.apache.iotdb.commons.schema.table.TsTable.TTL_PROPERTY; +import static org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.AbstractDatabaseTask.NEED_LAST_CACHE_KEY; import static org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.CreateDBTask.DATA_REGION_GROUP_NUM_KEY; import static org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.CreateDBTask.SCHEMA_REGION_GROUP_NUM_KEY; import static org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.CreateDBTask.TIME_PARTITION_INTERVAL_KEY; @@ -345,6 +348,11 @@ private IConfigTask visitDatabaseStatement( schema.setTTL(Long.MAX_VALUE); } break; + case NEED_LAST_CACHE_KEY: + if (node.getType() == DatabaseSchemaStatement.DatabaseSchemaStatementType.ALTER) { + schema.setNeedLastCache(true); + } + break; default: throw new SemanticException("Unsupported database property key: " + key); } @@ -378,6 +386,9 @@ private IConfigTask visitDatabaseStatement( case DATA_REGION_GROUP_NUM_KEY: schema.setMinDataRegionGroupNum(parseIntFromLiteral(value, DATA_REGION_GROUP_NUM_KEY)); break; + case NEED_LAST_CACHE_KEY: + schema.setNeedLastCache(parseBooleanFromLiteral(value, NEED_LAST_CACHE_KEY)); + break; default: throw new SemanticException("Unsupported database property key: " + key); } @@ -840,17 +851,26 @@ private Map convertPropertiesToMap( if (TABLE_ALLOWED_PROPERTIES.contains(key)) { if (!property.isSetToDefault()) { final Expression value = property.getNonDefaultValue(); - final Optional strValue = parseStringFromLiteralIfBinary(value); - if (strValue.isPresent()) { - if (!strValue.get().equalsIgnoreCase(TTL_INFINITE)) { - throw new SemanticException( - "ttl value must be 'INF' or a long literal, but now is: " + value); - } - map.put(key, strValue.get().toUpperCase(Locale.ENGLISH)); - continue; + switch (key) { + case TTL_PROPERTY: + final Optional strValue = parseStringFromLiteralIfBinary(value); + if (strValue.isPresent()) { + if (!strValue.get().equalsIgnoreCase(TTL_INFINITE)) { + throw new SemanticException( + "ttl value must be 'INF' or a long literal, but now is: " + value); + } + map.put(key, strValue.get().toUpperCase(Locale.ENGLISH)); + continue; + } + map.put(key, String.valueOf(parseLongFromLiteral(value, TTL_PROPERTY))); + break; + case NEED_LAST_CACHE_PROPERTY: + map.put( + key, String.valueOf(parseBooleanFromLiteral(value, NEED_LAST_CACHE_PROPERTY))); + break; + default: + break; } - // TODO: support validation for other properties - map.put(key, String.valueOf(parseLongFromLiteral(value, TTL_PROPERTY))); } else if (serializeDefault) { map.put(key, null); } @@ -1027,6 +1047,18 @@ protected IConfigTask visitSetSystemStatus(SetSystemStatus node, MPPQueryContext return new SetSystemStatusTask(((SetSystemStatusStatement) node.getInnerTreeStatement())); } + private boolean parseBooleanFromLiteral(final Object value, final String name) { + if (!(value instanceof BooleanLiteral)) { + throw new SemanticException( + name + + " value must be a BooleanLiteral, but now is " + + (Objects.nonNull(value) ? value.getClass().getSimpleName() : null) + + ", value: " + + value); + } + return ((BooleanLiteral) value).getValue(); + } + private Optional parseStringFromLiteralIfBinary(final Object value) { return value instanceof Literal && ((Literal) value).getTsValue() instanceof Binary ? Optional.of( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/DatabaseSchemaTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/DatabaseSchemaTask.java index ab9dee095aaa..a6eef0fa365f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/DatabaseSchemaTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/DatabaseSchemaTask.java @@ -72,6 +72,7 @@ public static TDatabaseSchema constructDatabaseSchema( if (databaseSchemaStatement.getDataRegionGroupNum() != null) { databaseSchema.setMinDataRegionGroupNum(databaseSchemaStatement.getDataRegionGroupNum()); } + databaseSchema.setNeedLastCache(databaseSchemaStatement.isNeedLastCache()); databaseSchema.setIsTableModel(false); return databaseSchema; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/AbstractDatabaseTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/AbstractDatabaseTask.java index df280c8759de..b4e0fa355673 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/AbstractDatabaseTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/AbstractDatabaseTask.java @@ -29,6 +29,7 @@ public abstract class AbstractDatabaseTask implements IConfigTask { public static final String TIME_PARTITION_INTERVAL_KEY = "time_partition_interval"; public static final String SCHEMA_REGION_GROUP_NUM_KEY = "schema_region_group_num"; public static final String DATA_REGION_GROUP_NUM_KEY = "data_region_group_num"; + public static final String NEED_LAST_CACHE_KEY = "need_last_cache"; // Deprecated public static final String SCHEMA_REPLICATION_FACTOR_KEY = "schema_replication_factor"; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/ShowCreateTableTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/ShowCreateTableTask.java index 64581425f01b..2cf0bf80cbab 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/ShowCreateTableTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/ShowCreateTableTask.java @@ -134,6 +134,8 @@ private static String getShowCreateTableSQL(final TsTable table) { builder .append(" WITH (ttl=") .append(table.getPropValue(TsTable.TTL_PROPERTY).orElse("'" + TTL_INFINITE + "'")) + .append(", need_last_cache=") + .append(table.getPropValue(TsTable.NEED_LAST_CACHE_PROPERTY).orElse("true")) .append(")"); return builder.toString(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/ShowCreateViewTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/ShowCreateViewTask.java index a0c8325b8643..2674c5eaa2c6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/ShowCreateViewTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/ShowCreateViewTask.java @@ -139,6 +139,8 @@ public static String getShowCreateViewSQL(final TsTable table) { builder .append(" WITH (ttl=") .append(table.getPropValue(TsTable.TTL_PROPERTY).orElse("'" + TTL_INFINITE + "'")) + .append(", need_last_cache=") + .append(table.getPropValue(TsTable.NEED_LAST_CACHE_PROPERTY).orElse("true")) .append(")"); builder.append(" AS "); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/ShowDBTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/ShowDBTask.java index f5ab58d3ead4..1db2128a0ae4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/ShowDBTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/ShowDBTask.java @@ -167,6 +167,7 @@ private static void buildTSBlockForDetails( builder.getColumnBuilder(4).writeLong(storageGroupInfo.getTimePartitionInterval()); builder.getColumnBuilder(5).writeInt(storageGroupInfo.getSchemaRegionNum()); builder.getColumnBuilder(6).writeInt(storageGroupInfo.getDataRegionNum()); + builder.getColumnBuilder(7).writeBoolean(storageGroupInfo.isNeedLastCache()); builder.declarePosition(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/ShowTablesDetailsTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/ShowTablesDetailsTask.java index 228b1dd266ac..2ddf07f9e0ff 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/ShowTablesDetailsTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/ShowTablesDetailsTask.java @@ -104,6 +104,9 @@ public static void buildTsBlock( ? TableType.values()[tableInfo.getType()].getName() : TableType.BASE_TABLE.getName(), TSFileConfig.STRING_CHARSET)); + builder + .getColumnBuilder(5) + .writeBoolean(!tableInfo.isSetNeedLastCache() || tableInfo.isNeedLastCache()); builder.declarePosition(); }); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java index 9edffa2e9e94..cbe23f35aacb 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java @@ -2878,6 +2878,13 @@ private void parseDatabaseAttributesClause( ctx.databaseAttributeClause()) { final IoTDBSqlParser.DatabaseAttributeKeyContext attributeKey = attribute.databaseAttributeKey(); + if (attributeKey == null) { + if (attribute.NEED_LAST_CACHE() != null) { + databaseSchemaStatement.setNeedLastCache( + Boolean.parseBoolean(attribute.boolean_literal().getText())); + } + continue; + } if (attributeKey.TTL() != null) { final long ttl = Long.parseLong(attribute.INTEGER_LITERAL().getText()); databaseSchemaStatement.setTtl(ttl); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java index 7786876e19ad..1d352c6a64b5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java @@ -65,6 +65,7 @@ import org.apache.iotdb.udf.api.relational.ScalarFunction; import org.apache.iotdb.udf.api.relational.TableFunction; +import org.apache.tsfile.annotations.TableModel; import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.read.common.type.ObjectType; import org.apache.tsfile.read.common.type.StringType; @@ -1462,6 +1463,7 @@ public SchemaPartition getOrCreateSchemaPartition( return partitionFetcher.getOrCreateSchemaPartition(database, deviceIDList, userName); } + @TableModel @Override public SchemaPartition getSchemaPartition( final String database, final List deviceIDList) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TableDeviceSchemaCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TableDeviceSchemaCache.java index ebbb137a2d83..a19948832914 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TableDeviceSchemaCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TableDeviceSchemaCache.java @@ -25,6 +25,7 @@ import org.apache.iotdb.commons.path.ExtendedPartialPath; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.path.PathPatternUtil; +import org.apache.iotdb.commons.schema.table.TsTable; import org.apache.iotdb.commons.service.metric.MetricService; import org.apache.iotdb.commons.utils.PathUtils; import org.apache.iotdb.db.conf.DataNodeMemoryConfig; @@ -234,8 +235,9 @@ public void initOrInvalidateLastCache( readWriteLock.readLock().lock(); try { // Avoid stale table - if (Objects.isNull( - DataNodeTableCache.getInstance().getTable(database, deviceId.getTableName(), false))) { + final TsTable table = + DataNodeTableCache.getInstance().getTable(database, deviceId.getTableName(), false); + if (Objects.isNull(table) || Boolean.FALSE.equals(table.getCachedNeedLastCache())) { return; } dualKeyCache.update( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TreeDeviceSchemaCacheManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TreeDeviceSchemaCacheManager.java index cdb66fdc0474..207dfbc66503 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TreeDeviceSchemaCacheManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TreeDeviceSchemaCacheManager.java @@ -28,6 +28,7 @@ import org.apache.iotdb.db.exception.metadata.view.InsertNonWritableViewException; import org.apache.iotdb.db.queryengine.common.schematree.ClusterSchemaTree; import org.apache.iotdb.db.queryengine.common.schematree.IMeasurementSchemaInfo; +import org.apache.iotdb.db.queryengine.plan.analyze.ClusterPartitionFetcher; import org.apache.iotdb.db.queryengine.plan.analyze.schema.ISchemaComputation; import org.apache.iotdb.db.schemaengine.template.ClusterTemplateManager; import org.apache.iotdb.db.schemaengine.template.ITemplateManager; @@ -337,6 +338,10 @@ public void updateLastCacheIfExists( final @Nonnull TimeValuePair[] timeValuePairs, final boolean isAligned, final IMeasurementSchema[] measurementSchemas) { + if (!ClusterPartitionFetcher.getInstance().needLastCache(database)) { + return; + } + tableDeviceSchemaCache.updateLastCache( database, deviceID, measurements, timeValuePairs, isAligned, measurementSchemas, false); } @@ -361,6 +366,10 @@ public void updateLastCacheIfExists( * @param measurementPath the fetched {@link MeasurementPath} */ public void declareLastCache(final String database, final MeasurementPath measurementPath) { + if (!ClusterPartitionFetcher.getInstance().needLastCache(database)) { + return; + } + tableDeviceSchemaCache.updateLastCache( database, measurementPath.getIDeviceID(), diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/DatabaseSchemaStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/DatabaseSchemaStatement.java index 53ff33e2437c..0b424fc08ed3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/DatabaseSchemaStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/DatabaseSchemaStatement.java @@ -39,6 +39,7 @@ public class DatabaseSchemaStatement extends Statement implements IConfigStateme private Integer schemaRegionGroupNum = null; private Integer dataRegionGroupNum = null; private boolean enablePrintExceptionLog = true; + private boolean needLastCache = true; // Deprecated private Integer schemaReplicationFactor = null; @@ -118,6 +119,14 @@ public void setEnablePrintExceptionLog(final boolean enablePrintExceptionLog) { this.enablePrintExceptionLog = enablePrintExceptionLog; } + public boolean isNeedLastCache() { + return needLastCache; + } + + public void setNeedLastCache(final boolean needLastCache) { + this.needLastCache = needLastCache; + } + @Override public R accept(final StatementVisitor visitor, final C context) { switch (subType) { @@ -141,8 +150,8 @@ public List getPaths() { @Override public String toString() { - return "SetStorageGroupStatement{" - + "storageGroupPath=" + return "DatabaseSchemaStatement{" + + "databasePath=" + databasePath + ", ttl=" + ttl diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/ShowDatabaseStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/ShowDatabaseStatement.java index e1a536622429..cda4b552946c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/ShowDatabaseStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/ShowDatabaseStatement.java @@ -106,6 +106,7 @@ public void buildTSBlock( builder.getColumnBuilder(8).writeInt(databaseInfo.getDataRegionNum()); builder.getColumnBuilder(9).writeInt(databaseInfo.getMinDataRegionNum()); builder.getColumnBuilder(10).writeInt(databaseInfo.getMaxDataRegionNum()); + builder.getColumnBuilder(11).writeBoolean(databaseInfo.isNeedLastCache()); } builder.declarePosition(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/InformationSchemaUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/InformationSchemaUtils.java index 337b0ffd78fe..9841576a3a38 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/InformationSchemaUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/InformationSchemaUtils.java @@ -75,6 +75,7 @@ public static void buildDatabaseTsBlock( if (details) { builder.getColumnBuilder(5).appendNull(); builder.getColumnBuilder(6).appendNull(); + builder.getColumnBuilder(7).writeBoolean(false); } builder.declarePosition(); } @@ -121,6 +122,7 @@ public static boolean mayShowTable( builder .getColumnBuilder(4) .writeBinary(new Binary(TableType.SYSTEM_VIEW.getName(), TSFileConfig.STRING_CHARSET)); + builder.getColumnBuilder(5).writeBoolean(false); } builder.declarePosition(); } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/PartitionCacheTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/PartitionCacheTest.java index e08a2b610c99..7f5f41ba86d0 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/PartitionCacheTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/PartitionCacheTest.java @@ -62,7 +62,7 @@ public class PartitionCacheTest { SeriesPartitionExecutor.getSeriesPartitionExecutor( config.getSeriesPartitionExecutorClass(), config.getSeriesPartitionSlotNum()); - private static final Set storageGroups = new HashSet<>(); + private static final Set databases = new HashSet<>(); private static final Map> schemaPartitionTable = new HashMap<>(); private static final Map< @@ -85,7 +85,7 @@ public class PartitionCacheTest { storageGroupNumber++) { // init each database String storageGroupName = getDatabaseName(storageGroupNumber); - storageGroups.add(storageGroupName); + databases.add(storageGroupName); if (!schemaPartitionTable.containsKey(storageGroupName)) { schemaPartitionTable.put(storageGroupName, new HashMap<>()); } @@ -148,7 +148,7 @@ private static String getDeviceName(String storageGroupName, int deviceNumber) { @Before public void setUp() throws Exception { partitionCache = new PartitionCache(); - partitionCache.updateDatabaseCache(storageGroups); + partitionCache.updateDatabaseCache(databases); partitionCache.updateSchemaPartitionCache(schemaPartitionTable); partitionCache.updateDataPartitionCache(dataPartitionTable); partitionCache.updateGroupIdToReplicaSetMap(100, consensusGroupIdToRegionReplicaSet); diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/column/ColumnHeaderConstant.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/column/ColumnHeaderConstant.java index 0459d4d2c86e..262f64520e47 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/column/ColumnHeaderConstant.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/column/ColumnHeaderConstant.java @@ -236,6 +236,7 @@ private ColumnHeaderConstant() { public static final String TIME_PARTITION_INTERVAL_TABLE_MODEL = "time_partition_interval"; public static final String SCHEMA_REGION_GROUP_NUM_TABLE_MODEL = "schema_region_group_num"; public static final String DATA_REGION_GROUP_NUM_TABLE_MODEL = "data_region_group_num"; + public static final String NEED_LAST_CACHE_TABLE_MODEL = "need_last_cache"; public static final String REGION_ID_TABLE_MODEL = "region_id"; public static final String DATANODE_ID_TABLE_MODEL = "datanode_id"; @@ -322,6 +323,7 @@ private ColumnHeaderConstant() { public static final String PRIVILEGES = "Privileges"; public static final String COMMENT = "Comment"; public static final String TABLE_TYPE = "TableType"; + public static final String NEED_LAST_CACHE = "NeedLastCache"; public static final String VIEW = "View"; public static final String CREATE_VIEW = "Create View"; @@ -399,7 +401,8 @@ private ColumnHeaderConstant() { new ColumnHeader(MAX_SCHEMA_REGION_GROUP_NUM, TSDataType.INT32), new ColumnHeader(DATA_REGION_GROUP_NUM, TSDataType.INT32), new ColumnHeader(MIN_DATA_REGION_GROUP_NUM, TSDataType.INT32), - new ColumnHeader(MAX_DATA_REGION_GROUP_NUM, TSDataType.INT32)); + new ColumnHeader(MAX_DATA_REGION_GROUP_NUM, TSDataType.INT32), + new ColumnHeader(NEED_LAST_CACHE, TSDataType.BOOLEAN)); public static final List showChildPathsColumnHeaders = ImmutableList.of( @@ -688,7 +691,8 @@ private ColumnHeaderConstant() { new ColumnHeader(DATA_REPLICATION_FACTOR, TSDataType.INT32), new ColumnHeader(TIME_PARTITION_INTERVAL, TSDataType.INT64), new ColumnHeader(SCHEMA_REGION_GROUP_NUM, TSDataType.INT32), - new ColumnHeader(DATA_REGION_GROUP_NUM, TSDataType.INT32)); + new ColumnHeader(DATA_REGION_GROUP_NUM, TSDataType.INT32), + new ColumnHeader(NEED_LAST_CACHE, TSDataType.BOOLEAN)); public static final List describeTableColumnHeaders = ImmutableList.of( @@ -728,7 +732,8 @@ private ColumnHeaderConstant() { new ColumnHeader(COLUMN_TTL, TSDataType.TEXT), new ColumnHeader(STATUS, TSDataType.TEXT), new ColumnHeader(COMMENT, TSDataType.TEXT), - new ColumnHeader(TABLE_TYPE, TSDataType.TEXT)); + new ColumnHeader(TABLE_TYPE, TSDataType.TEXT), + new ColumnHeader(NEED_LAST_CACHE, TSDataType.BOOLEAN)); public static final List LIST_USER_OR_ROLE_PRIVILEGES_COLUMN_HEADERS = ImmutableList.of( diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/InformationSchema.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/InformationSchema.java index 4f458d34e05a..df475b783910 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/InformationSchema.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/InformationSchema.java @@ -93,6 +93,9 @@ public class InformationSchema { databaseTable.addColumnSchema( new AttributeColumnSchema( ColumnHeaderConstant.DATA_REGION_GROUP_NUM_TABLE_MODEL, TSDataType.INT32)); + databaseTable.addColumnSchema( + new AttributeColumnSchema( + ColumnHeaderConstant.NEED_LAST_CACHE_TABLE_MODEL, TSDataType.BOOLEAN)); databaseTable.removeColumnSchema(TsTable.TIME_COLUMN_NAME); schemaTables.put(DATABASES, databaseTable); @@ -113,6 +116,9 @@ public class InformationSchema { ColumnHeaderConstant.COMMENT.toLowerCase(Locale.ENGLISH), TSDataType.STRING)); tableTable.addColumnSchema( new AttributeColumnSchema(ColumnHeaderConstant.TABLE_TYPE_TABLE_MODEL, TSDataType.STRING)); + tableTable.addColumnSchema( + new AttributeColumnSchema( + ColumnHeaderConstant.NEED_LAST_CACHE_TABLE_MODEL, TSDataType.BOOLEAN)); tableTable.removeColumnSchema(TsTable.TIME_COLUMN_NAME); schemaTables.put(TABLES, tableTable); diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TsTable.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TsTable.java index ae7fb8e38c93..b9c9ec82d726 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TsTable.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TsTable.java @@ -45,8 +45,10 @@ import java.io.OutputStream; import java.nio.ByteBuffer; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -70,7 +72,10 @@ public class TsTable { new TimeColumnSchema(TIME_COLUMN_NAME, TSDataType.TIMESTAMP); public static final String TTL_PROPERTY = "ttl"; - public static final Set TABLE_ALLOWED_PROPERTIES = Collections.singleton(TTL_PROPERTY); + public static final String NEED_LAST_CACHE_PROPERTY = "need_last_cache"; + public static final Set TABLE_ALLOWED_PROPERTIES = + Collections.unmodifiableSet( + new HashSet<>(Arrays.asList(TTL_PROPERTY, NEED_LAST_CACHE_PROPERTY))); private static final String OBJECT_STRING_ERROR = "When there are object fields, the %s %s shall not be '.', '..' or contain './', '.\\'"; protected String tableName; @@ -97,6 +102,7 @@ public class TsTable { // Cache, avoid string parsing private transient long ttlValue = Long.MIN_VALUE; + private transient Boolean needLastCache = null; private transient int tagNums = 0; private transient int fieldNum = 0; @@ -315,6 +321,14 @@ public long getTableTTL() { : Long.MAX_VALUE; } + public boolean getCachedNeedLastCache() { + if (needLastCache == null) { + needLastCache = + getPropValue(NEED_LAST_CACHE_PROPERTY).map(Boolean::parseBoolean).orElse(true); + } + return needLastCache; + } + public Map getProps() { readWriteLock.readLock().lock(); try { diff --git a/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift b/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift index 01bde7c378dc..385448ca9c60 100644 --- a/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift +++ b/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift @@ -226,6 +226,7 @@ struct TDatabaseSchema { 9: optional i32 maxDataRegionGroupNum 10: optional i64 timePartitionOrigin 11: optional bool isTableModel + 12: optional bool needLastCache } // Schema @@ -722,6 +723,7 @@ struct TDatabaseInfo { 10: required i32 minDataRegionNum 11: required i32 maxDataRegionNum 12: optional i64 timePartitionOrigin + 13: optional bool needLastCache } struct TGetDatabaseReq { @@ -1247,6 +1249,7 @@ struct TTableInfo { 3: optional i32 state 4: optional string comment 5: optional i32 type + 6: optional bool needLastCache } struct TCreateTableViewReq {