From 3dfc42c06afe6e99373d5642c4e2ea57ab7fd4f7 Mon Sep 17 00:00:00 2001 From: "yonghao.fyh" Date: Mon, 19 Oct 2026 14:09:18 +0800 Subject: [PATCH 1/6] feat(table): make Table a resolved handle and the home of operator creation --- examples/read_write_demo.cpp | 14 +- include/paimon/catalog/identifier.h | 18 +- include/paimon/catalog/table.h | 90 ++++- include/paimon/table/format/format_table.h | 2 +- include/paimon/type_fwd.h | 3 + .../common/data/blob_view_struct_test.cpp | 7 +- src/paimon/common/data/blob_view_test.cpp | 3 +- .../core/catalog/file_system_catalog.cpp | 21 +- .../core/catalog/file_system_catalog_test.cpp | 21 +- src/paimon/core/catalog/identifier.cpp | 52 +-- src/paimon/core/catalog/identifier_test.cpp | 51 +-- .../operation/file_store_commit_impl_test.cpp | 4 +- src/paimon/core/operation/read_context.cpp | 8 +- .../core/table/format/format_table_loader.cpp | 3 +- src/paimon/core/table/system/system_table.cpp | 3 +- src/paimon/core/table/table.cpp | 83 ++++- src/paimon/core/table/table_test.cpp | 308 +++++++++++++++++- src/paimon/core/utils/branch_manager_test.cpp | 2 +- src/paimon/rest/rest_api.cpp | 18 +- src/paimon/rest/rest_catalog.cpp | 44 ++- src/paimon/rest/rest_catalog.h | 4 +- src/paimon/rest/rest_catalog_test.cpp | 6 + src/paimon/testing/mock/mock_catalog.h | 8 +- 23 files changed, 632 insertions(+), 141 deletions(-) diff --git a/examples/read_write_demo.cpp b/examples/read_write_demo.cpp index bcf7db1e1..df46fcc54 100644 --- a/examples/read_write_demo.cpp +++ b/examples/read_write_demo.cpp @@ -26,6 +26,7 @@ #include "arrow/ipc/api.h" #include "paimon/api.h" #include "paimon/catalog/catalog.h" +#include "paimon/catalog/table.h" arrow::Result> PrepareData(const arrow::FieldVector& fields) { arrow::StringBuilder f0_builder; @@ -81,6 +82,11 @@ paimon::Status Run(const std::string& root_path, const std::string& db_name, /*primary_keys=*/{}, options, /*ignore_if_exists=*/false)); + // The table every operator below is created from. Each call hands the table's shared owner + // to the operator it creates. + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table, + catalog->GetTable(paimon::Identifier(db_name, table_name))); + std::string table_path = root_path + "/" + db_name + ".db/" + table_name; std::string commit_user = "some_commit_user"; @@ -89,7 +95,7 @@ paimon::Status Run(const std::string& root_path, const std::string& db_name, PAIMON_ASSIGN_OR_RAISE(std::unique_ptr write_context, context_builder.SetOptions(options).Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr writer, - paimon::FileStoreWrite::Create(std::move(write_context))); + table->NewWrite(std::move(write_context))); // prepare data auto struct_array = PrepareData(fields); @@ -113,7 +119,7 @@ paimon::Status Run(const std::string& root_path, const std::string& db_name, PAIMON_ASSIGN_OR_RAISE(std::unique_ptr commit_context, commit_context_builder.SetOptions(options).Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr committer, - paimon::FileStoreCommit::Create(std::move(commit_context))); + table->NewCommit(std::move(commit_context))); PAIMON_RETURN_NOT_OK(committer->Commit(commit_message)); // scan @@ -121,7 +127,7 @@ paimon::Status Run(const std::string& root_path, const std::string& db_name, PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan_context, scan_context_builder.SetOptions(options).Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scanner, - paimon::TableScan::Create(std::move(scan_context))); + table->NewScan(std::move(scan_context))); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr plan, scanner->CreatePlan()); // read @@ -129,7 +135,7 @@ paimon::Status Run(const std::string& root_path, const std::string& db_name, PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, read_context_builder.SetOptions(options).Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_read, - paimon::TableRead::Create(std::move(read_context))); + table->NewRead(std::move(read_context))); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr batch_reader, table_read->CreateReader(plan->Splits())); arrow::ArrayVector result_array_vector; diff --git a/include/paimon/catalog/identifier.h b/include/paimon/catalog/identifier.h index bcc06208a..af7e39981 100644 --- a/include/paimon/catalog/identifier.h +++ b/include/paimon/catalog/identifier.h @@ -27,7 +27,8 @@ namespace paimon { -/// An identifier for a table containing database and table name. +/// An identifier for a catalog object: a database name plus an object name, which may carry a +/// branch or system-table suffix. class PAIMON_EXPORT Identifier { public: static const char kUnknownDatabase[]; @@ -35,8 +36,7 @@ class PAIMON_EXPORT Identifier { static const char kSystemBranchPrefix[]; static const char kDefaultMainBranch[]; - explicit Identifier(const std::string& table); - Identifier(const std::string& database, const std::string& table); + Identifier(const std::string& database, const std::string& object); /// Names `branch` of `table`: every branch but the main one is addressed by the object name /// this builds, which joins them as `tbl$branch_dev`. /// @@ -53,9 +53,15 @@ class PAIMON_EXPORT Identifier { bool operator==(const Identifier& other) const; const std::string& GetDatabaseName() const; - const std::string& GetTableName() const; + /// The full object name, which keeps any branch and system-table suffix, like + /// `tbl$branch_dev`. Mirrors the Java `Identifier#getObjectName`. + const std::string& GetObjectName() const; Result GetDataTableName() const; Result> GetBranchName() const; + /// The branch the object name carries, `main` when it names none. Any spelling of the + /// main branch, as `MAIN`, folds to `main` here, as the branch constructor folds it, + /// so a caller binding a manager to the returned value addresses the main branch's own + /// directory rather than one named after the alias. Result GetBranchNameOrDefault() const; Result> GetSystemTableName() const; Result IsSystemTable() const; @@ -67,10 +73,10 @@ class PAIMON_EXPORT Identifier { static Result FromString(const std::string& full_name); private: - Status SplitTableName() const; + Status SplitObjectName() const; const std::string database_; - const std::string table_; + const std::string object_; mutable bool parsed_ = false; mutable std::string data_table_; mutable std::optional branch_; diff --git a/include/paimon/catalog/table.h b/include/paimon/catalog/table.h index 59c82495f..2194355c8 100644 --- a/include/paimon/catalog/table.h +++ b/include/paimon/catalog/table.h @@ -33,6 +33,9 @@ namespace paimon { +class TableScan; +class TableRead; + /// A table provides basic abstraction for table type. class PAIMON_EXPORT Table { public: @@ -46,20 +49,64 @@ class PAIMON_EXPORT Table { const std::string& table_path, const Identifier& identifier); + /// @param file_system File system the table's data is accessed through. + /// @param root_path Root path of the table in the file system. /// @param uuid Metastore UUID, or empty if unavailable. - Table(const std::shared_ptr& schema, const std::string& database, - const std::string& table_name, const std::string& uuid = "") - : schema_(schema), database_(database), table_name_(table_name), uuid_(uuid) {} + /// @param schema_manager A schema manager the table keeps instead of building its own, + /// holding the schema cache an opener already warmed; built when null. + Table(const std::shared_ptr& file_system, const std::string& root_path, + const std::shared_ptr& schema, const Identifier& identifier, + const std::string& uuid = "", + const std::shared_ptr& schema_manager = nullptr); ~Table() = default; + /// Returns the file system the table was opened with, which serves the table's + /// files. A catalog that hands out per-table file systems opens a table with the + /// one for that table; it is null when the opener had none. + std::shared_ptr GetFileSystem() const { + return file_system_; + } + + /// Returns the root path of the table in its file system. It is empty when the + /// opener had no location for the table, which a mock in a test is one case of. + const std::string& GetRootPath() const { + return root_path_; + } + + /// Returns the manager of the table's schema versions, bound to the table's file + /// system, root path and branch, so schema files read through one table are not + /// read again through another. It is null when the table was opened without a + /// file system or with an identifier whose branch cannot be parsed, which a mock + /// in a test is one case of. + std::shared_ptr GetSchemaManager() const { + return schema_manager_; + } + + /// Returns the manager of the table's snapshots, bound like the schema manager. + /// It carries no snapshot loader: an operator that needs one attaches it to a + /// manager of its own, since a loader attached here would leak into every read + /// through this table. + std::shared_ptr GetSnapshotManager() const { + return snapshot_manager_; + } + /// A name to identify this table. std::string Name() const { - return table_name_; + return identifier_.GetObjectName(); } /// Full name of the table, default is database.tableName. - std::string FullName() const; + std::string FullName() const { + return identifier_.GetFullName(); + } + + /// Returns the branch the table was opened for, which its identifier carries; `main` + /// when it names none or a spelling of it, which its managers are bound to. A + /// malformed object name that hides the branch is reported. + Result CurrentBranch() const { + return identifier_.GetBranchNameOrDefault(); + } /// Returns the metastore UUID, falling back to `FullName()`. /// @note Use `CatalogUuid()` for commit requests to avoid sending a table name as an ID. @@ -78,11 +125,40 @@ class PAIMON_EXPORT Table { return schema_; } + /// Creates a scan of this table from a finished scan context, the table-first entry point + /// that `TableScan::Create()` also builds through. The scan is derived from the context. + /// @param context The finished scan context. + /// @return The scan, or the error building it reports. + Result> NewScan(std::unique_ptr context) const; + + /// Creates a read of this table from a finished read context, the table-first entry point + /// that `TableRead::Create()` also builds through. The read is derived from the context. + /// @param context The finished read context. + /// @return The read, or the error building it reports. + Result> NewRead(std::unique_ptr context) const; + + /// Creates a write of this table from a finished write context, the table-first entry point + /// that `FileStoreWrite::Create()` also builds through. The write is derived from the context. + /// @param context The finished write context. + /// @return The write, or the error building it reports. + Result> NewWrite(std::unique_ptr context) const; + + /// Creates a commit of this table from a finished commit context, the table-first entry point + /// that `FileStoreCommit::Create()` also builds through. The commit is derived from the + /// context. + /// @param context The finished commit context. + /// @return The commit, or the error building it reports. + Result> NewCommit( + std::unique_ptr context) const; + private: + std::shared_ptr file_system_; + std::string root_path_; std::shared_ptr schema_; - std::string database_; - std::string table_name_; + Identifier identifier_; std::string uuid_; + std::shared_ptr schema_manager_; + std::shared_ptr snapshot_manager_; }; } // namespace paimon diff --git a/include/paimon/table/format/format_table.h b/include/paimon/table/format/format_table.h index 3f07880e4..a4c310254 100644 --- a/include/paimon/table/format/format_table.h +++ b/include/paimon/table/format/format_table.h @@ -157,7 +157,7 @@ class PAIMON_EXPORT FormatTable { /// A name to identify this table. std::string Name() const { - return identifier_.GetTableName(); + return identifier_.GetObjectName(); } /// Full name of the table, database.tableName. diff --git a/include/paimon/type_fwd.h b/include/paimon/type_fwd.h index 77efb108f..f2c93a0bb 100644 --- a/include/paimon/type_fwd.h +++ b/include/paimon/type_fwd.h @@ -75,6 +75,9 @@ class FileStoreCommit; class WriteContext; class CommitContext; +class SchemaManager; +class SnapshotManager; + class CommitMessage; class Metrics; diff --git a/src/paimon/common/data/blob_view_struct_test.cpp b/src/paimon/common/data/blob_view_struct_test.cpp index 9668b7758..caa6cb9e1 100644 --- a/src/paimon/common/data/blob_view_struct_test.cpp +++ b/src/paimon/common/data/blob_view_struct_test.cpp @@ -40,7 +40,7 @@ class BlobViewStructTest : public testing::Test { TEST_F(BlobViewStructTest, TestConstructorAndGetters) { ASSERT_EQ(view_struct_.GetIdentifier().GetDatabaseName(), "test_db"); - ASSERT_EQ(view_struct_.GetIdentifier().GetTableName(), "test_table"); + ASSERT_EQ(view_struct_.GetIdentifier().GetObjectName(), "test_table"); ASSERT_EQ(view_struct_.FieldId(), 7); ASSERT_EQ(view_struct_.RowId(), 1024); } @@ -53,7 +53,7 @@ TEST_F(BlobViewStructTest, TestSerializeDeserializeRoundTrip) { ASSERT_OK_AND_ASSIGN(auto restored, BlobViewStruct::Deserialize(serialized->data(), serialized->size())); ASSERT_EQ(restored->GetIdentifier().GetDatabaseName(), "test_db"); - ASSERT_EQ(restored->GetIdentifier().GetTableName(), "test_table"); + ASSERT_EQ(restored->GetIdentifier().GetObjectName(), "test_table"); ASSERT_EQ(restored->FieldId(), 7); ASSERT_EQ(restored->RowId(), 1024); } @@ -78,7 +78,8 @@ TEST_F(BlobViewStructTest, TestToString) { } TEST_F(BlobViewStructTest, TestSerializeRejectsUnknownDatabase) { - BlobViewStruct view_struct(Identifier("test_table"), /*field_id=*/7, /*row_id=*/1024); + BlobViewStruct view_struct(Identifier(Identifier::kUnknownDatabase, "test_table"), + /*field_id=*/7, /*row_id=*/1024); ASSERT_NOK_WITH_MSG(view_struct.Serialize(pool_), "upstream table identifier must include database name"); } diff --git a/src/paimon/common/data/blob_view_test.cpp b/src/paimon/common/data/blob_view_test.cpp index 380bec0ec..92ff8fe3e 100644 --- a/src/paimon/common/data/blob_view_test.cpp +++ b/src/paimon/common/data/blob_view_test.cpp @@ -80,7 +80,8 @@ TEST_F(BlobViewTest, TestJavaSerializationCompatibility) { } TEST_F(BlobViewTest, TestToViewRejectsInvalidArguments) { - BlobView unknown_database{Identifier("source"), /*field_id=*/7, /*row_id=*/5}; + BlobView unknown_database{Identifier(Identifier::kUnknownDatabase, "source"), + /*field_id=*/7, /*row_id=*/5}; ASSERT_NOK_WITH_MSG(BlobView::ToView(unknown_database, pool_), "upstream table identifier must include database name"); ASSERT_NOK_WITH_MSG(BlobView::ToView(blob_view_, /*pool=*/nullptr), "memory pool is nullptr"); diff --git a/src/paimon/core/catalog/file_system_catalog.cpp b/src/paimon/core/catalog/file_system_catalog.cpp index 827e50b4a..d150f34bb 100644 --- a/src/paimon/core/catalog/file_system_catalog.cpp +++ b/src/paimon/core/catalog/file_system_catalog.cpp @@ -103,7 +103,7 @@ Result FileSystemCatalog::DatabaseExists(const std::string& db_name) const Result FileSystemCatalog::TableExists(const Identifier& identifier) const { // Handle sys database global tables if (CatalogUtils::IsSystemDatabase(identifier.GetDatabaseName())) { - return GlobalSystemTableLoader::IsSupported(identifier.GetTableName(), catalog_options_); + return GlobalSystemTableLoader::IsSupported(identifier.GetObjectName(), catalog_options_); } // The system table suffix is dropped when the data table identifier is rebuilt below, so the // identifier is validated as a whole here. @@ -155,7 +155,7 @@ Status FileSystemCatalog::CreateTable(const Identifier& identifier, ArrowSchema* return Status::OK(); } else { return Status::Invalid( - fmt::format("table {} already exist", identifier.GetTableName())); + fmt::format("table {} already exist", identifier.GetObjectName())); } } PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr schema, @@ -182,10 +182,9 @@ Result>> FileSystemCatalog::TableSche "do not support checking TableSchemaExists for system table."); } PAIMON_ASSIGN_OR_RAISE(std::string table_path, NewDataTablePath(warehouse_, identifier)); + // Any spelling of the main branch folds to `main`, so the manager reads the schema + // of the branch the main alias spells. PAIMON_ASSIGN_OR_RAISE(std::string branch, identifier.GetBranchNameOrDefault()); - if (StringUtils::EqualsIgnoreCase(branch, Identifier::kDefaultMainBranch)) { - branch = Identifier::kDefaultMainBranch; - } SchemaManager schema_manager(fs_, table_path, branch); return schema_manager.Latest(); } @@ -281,7 +280,7 @@ Result> FileSystemCatalog::LoadTableSchema( // Handle sys database global tables if (CatalogUtils::IsSystemDatabase(identifier.GetDatabaseName())) { PAIMON_ASSIGN_OR_RAISE(bool supported, GlobalSystemTableLoader::IsSupported( - identifier.GetTableName(), catalog_options_)); + identifier.GetObjectName(), catalog_options_)); if (!supported) { return Status::NotExist(fmt::format("{} not exist", identifier.ToString())); } @@ -291,7 +290,7 @@ Result> FileSystemCatalog::LoadTableSchema( context.warehouse = warehouse_; context.catalog_options = catalog_options_; PAIMON_ASSIGN_OR_RAISE(std::shared_ptr system_table, - GlobalSystemTableLoader::Load(identifier.GetTableName(), context)); + GlobalSystemTableLoader::Load(identifier.GetObjectName(), context)); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr arrow_schema, system_table->ArrowSchema()); return std::make_shared(std::move(arrow_schema)); @@ -339,8 +338,12 @@ Result> FileSystemCatalog::GetTable(const Identifier& ide PAIMON_ASSIGN_OR_RAISE(bool is_system_table, identifier.IsSystemTable()); if (is_system_table) { PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, LoadTableSchema(identifier)); - return std::make_shared(schema, identifier.GetDatabaseName(), - identifier.GetTableName()); + // A system table reads the files of the data table it belongs to, so its root path + // is that table's. + PAIMON_ASSIGN_OR_RAISE(std::string data_table_name, identifier.GetDataTableName()); + Identifier data_identifier(identifier.GetDatabaseName(), data_table_name); + PAIMON_ASSIGN_OR_RAISE(std::string table_path, GetTableLocation(data_identifier)); + return std::make_shared
(fs_, table_path, schema, identifier); } PAIMON_ASSIGN_OR_RAISE(std::string table_path, GetTableLocation(identifier)); // `Table::Create` reads the same schema file and rejects a format table there. diff --git a/src/paimon/core/catalog/file_system_catalog_test.cpp b/src/paimon/core/catalog/file_system_catalog_test.cpp index 68aa6ac47..51e7c642d 100644 --- a/src/paimon/core/catalog/file_system_catalog_test.cpp +++ b/src/paimon/core/catalog/file_system_catalog_test.cpp @@ -224,6 +224,9 @@ TEST(FileSystemCatalogTest, TestOptionsSystemTableCatalog) { ASSERT_OK_AND_ASSIGN(auto system_table, catalog.GetTable(options_identifier)); ASSERT_EQ(system_table->Name(), "tbl1$options"); + // A system table reads the files of the data table it belongs to, so the table opened + // for it carries that table's root path. + ASSERT_EQ(system_table->GetRootPath(), table_path); ASSERT_NOK_WITH_MSG(catalog.LoadTableSchema(Identifier("db1", "tbl1$unknown")), "not exist"); ASSERT_NOK_WITH_MSG(catalog.LoadTableSchema(Identifier("db1", "missing$options")), "not exist"); @@ -617,9 +620,11 @@ TEST(FileSystemCatalogTest, TestCreateTableWithBlob) { arrow::ImportSchema(arrow_schema_from_get_table.get()).ValueOrDie(); ASSERT_TRUE(typed_schema.Equals(schema_from_get_table)); ASSERT_EQ(table->FullName(), "db1.tbl1"); + ASSERT_EQ(table->GetRootPath(), + PathUtil::JoinPath(PathUtil::JoinPath(dir->Str(), "db1.db"), "tbl1")); ASSERT_NOK_WITH_MSG(catalog.GetTable(Identifier("db1", "table_xaxa")), - "Identifier{database='db1', table='table_xaxa'} not exist"); + "Identifier{database='db1', object='table_xaxa'} not exist"); ArrowSchemaRelease(&schema); } @@ -832,7 +837,7 @@ TEST(FileSystemCatalogTest, TestValidateTableSchema) { /*ignore_if_exists=*/false)); ASSERT_NOK_WITH_MSG(catalog.LoadTableSchema(Identifier("db0", "tbl0")), - "Identifier{database=\'db0\', table=\'tbl0\'} not exist"); + "Identifier{database=\'db0\', object=\'tbl0\'} not exist"); ASSERT_OK_AND_ASSIGN(std::shared_ptr table_schema, catalog.LoadTableSchema(identifier)); auto data_schema = std::dynamic_pointer_cast(table_schema); ASSERT_TRUE(data_schema != nullptr); @@ -872,7 +877,7 @@ TEST(FileSystemCatalogTest, TestValidateTableSchema) { ASSERT_OK(fs->Delete(schema_path)); ASSERT_NOK_WITH_MSG(catalog.LoadTableSchema(identifier), - "Identifier{database=\'db1\', table=\'tbl1\'} not exist"); + "Identifier{database=\'db1\', object=\'tbl1\'} not exist"); ASSERT_NOK_WITH_MSG(catalog.LoadTableSchema(Identifier("db1", "tbl$11")), "not exist"); ArrowSchemaRelease(&schema); @@ -952,7 +957,7 @@ TEST(FileSystemCatalogTest, TestDropTable) { ASSERT_NOK_WITH_MSG( catalog.DropTable(Identifier("test_db", "non_existent_tbl"), /*ignore_if_not_exists=*/false), - "table Identifier{database='test_db', table='non_existent_tbl'} does not exist"); + "table Identifier{database='test_db', object='non_existent_tbl'} does not exist"); // Test 3: Drop existing table arrow::FieldVector fields = { @@ -971,7 +976,7 @@ TEST(FileSystemCatalogTest, TestDropTable) { ASSERT_NOK_WITH_MSG(catalog.DropTable(Identifier("test_db", "tbl$system"), /*ignore_if_not_exists=*/false), "Cannot 'dropTable' for system table 'Identifier{database='test_db', " - "table='tbl$system'}', please use data table."); + "object='tbl$system'}', please use data table."); ArrowSchemaRelease(&schema); } @@ -996,7 +1001,7 @@ TEST(FileSystemCatalogTest, TestRenameTable) { catalog.RenameTable(Identifier("test_db", "non_existent_tbl"), Identifier("test_db", "new_tbl"), /*ignore_if_not_exists=*/false), - "source table Identifier{database='test_db', table='non_existent_tbl'} does not exist"); + "source table Identifier{database='test_db', object='non_existent_tbl'} does not exist"); // Test 3: Normal rename arrow::FieldVector fields = { @@ -1023,7 +1028,7 @@ TEST(FileSystemCatalogTest, TestRenameTable) { ASSERT_NOK_WITH_MSG( catalog.RenameTable(Identifier("test_db", "new_tbl"), Identifier("test_db", "tbl2"), /*ignore_if_not_exists=*/false), - "target table Identifier{database='test_db', table='tbl2'} already exists"); + "target table Identifier{database='test_db', object='tbl2'} already exists"); // Test 5: Cross-database rename ASSERT_OK(catalog.CreateDatabase("test_db2", options, /*ignore_if_exists=*/false)); @@ -1626,7 +1631,7 @@ TEST(FileSystemCatalogTest, TestIdentifierNameValidationRules) { std::string table_error; }; const std::vector invalid_names = { - {"", "cannot be empty or whitespace", "Invalid table name"}, + {"", "cannot be empty or whitespace", "Invalid object name"}, {" ", "cannot be empty or whitespace", "cannot be empty or whitespace"}, {".", "cannot be '.' or '..'", "cannot be '.' or '..'"}, {"..", "cannot be '.' or '..'", "cannot be '.' or '..'"}, diff --git a/src/paimon/core/catalog/identifier.cpp b/src/paimon/core/catalog/identifier.cpp index 18fcc7975..fa95506c5 100644 --- a/src/paimon/core/catalog/identifier.cpp +++ b/src/paimon/core/catalog/identifier.cpp @@ -48,11 +48,8 @@ std::string BranchObjectName(const std::string& table, const std::string& branch } // namespace -Identifier::Identifier(const std::string& table) - : Identifier(std::string(kUnknownDatabase), table) {} - -Identifier::Identifier(const std::string& database, const std::string& table) - : database_(database), table_(table) {} +Identifier::Identifier(const std::string& database, const std::string& object) + : database_(database), object_(object) {} Identifier::Identifier(const std::string& database, const std::string& table, const std::string& branch) @@ -62,34 +59,37 @@ bool Identifier::operator==(const Identifier& other) const { if (this == &other) { return true; } - return database_ == other.database_ && table_ == other.table_; + return database_ == other.database_ && object_ == other.object_; } const std::string& Identifier::GetDatabaseName() const { return database_; } -const std::string& Identifier::GetTableName() const { - return table_; +const std::string& Identifier::GetObjectName() const { + return object_; } Result Identifier::GetDataTableName() const { - PAIMON_RETURN_NOT_OK(SplitTableName()); + PAIMON_RETURN_NOT_OK(SplitObjectName()); return data_table_; } Result> Identifier::GetBranchName() const { - PAIMON_RETURN_NOT_OK(SplitTableName()); + PAIMON_RETURN_NOT_OK(SplitObjectName()); return branch_; } Result Identifier::GetBranchNameOrDefault() const { PAIMON_ASSIGN_OR_RAISE(std::optional branch, GetBranchName()); - return branch.value_or(kDefaultMainBranch); + if (!branch || StringUtils::EqualsIgnoreCase(branch.value(), kDefaultMainBranch)) { + return std::string(kDefaultMainBranch); + } + return branch.value(); } Result> Identifier::GetSystemTableName() const { - PAIMON_RETURN_NOT_OK(SplitTableName()); + PAIMON_RETURN_NOT_OK(SplitObjectName()); return system_table_; } @@ -99,21 +99,21 @@ Result Identifier::IsSystemTable() const { } std::string Identifier::ToString() const { - return fmt::format("Identifier{{database='{}', table='{}'}}", database_, table_); + return fmt::format("Identifier{{database='{}', object='{}'}}", database_, object_); } int32_t Identifier::HashCode() const { int32_t hash = MurmurHashUtils::HashUnsafeBytes(reinterpret_cast(database_.data()), /*offset=*/0, database_.size()); - return MurmurHashUtils::HashUnsafeBytes(reinterpret_cast(table_.data()), - /*offset=*/0, table_.size(), hash); + return MurmurHashUtils::HashUnsafeBytes(reinterpret_cast(object_.data()), + /*offset=*/0, object_.size(), hash); } std::string Identifier::GetFullName() const { if (database_ == kUnknownDatabase) { - return table_; + return object_; } - return fmt::format("{}.{}", database_, table_); + return fmt::format("{}.{}", database_, object_); } Result Identifier::FromString(const std::string& full_name) { @@ -124,14 +124,14 @@ Result Identifier::FromString(const std::string& full_name) { const auto dot_pos = full_name.find('.'); if (dot_pos == std::string::npos || dot_pos == 0 || dot_pos == full_name.size() - 1) { return Status::Invalid( - fmt::format("cannot get splits from '{}' to get database and table", full_name)); + fmt::format("cannot get splits from '{}' to get database and object", full_name)); } std::string database = full_name.substr(0, dot_pos); - std::string table = full_name.substr(dot_pos + 1); - return Identifier(database, table); + std::string object = full_name.substr(dot_pos + 1); + return Identifier(database, object); } -Status Identifier::SplitTableName() const { +Status Identifier::SplitObjectName() const { if (parsed_) { return Status::OK(); } @@ -139,9 +139,9 @@ Status Identifier::SplitTableName() const { std::optional branch; std::optional system_table; std::vector splits = - StringUtils::Split(table_, kSystemTableSplitter, /*ignore_empty=*/false); + StringUtils::Split(object_, kSystemTableSplitter, /*ignore_empty=*/false); if (splits.size() == 1) { - data_table = table_; + data_table = object_; } else if (splits.size() == 2) { data_table = splits[0]; if (StringUtils::StartsWith(splits[1], kSystemBranchPrefix, /*start_pos=*/0)) { @@ -152,20 +152,20 @@ Status Identifier::SplitTableName() const { } else if (splits.size() == 3) { if (!StringUtils::StartsWith(splits[1], kSystemBranchPrefix, /*start_pos=*/0)) { return Status::Invalid(fmt::format( - "System table can only contain one '$' separator, but this is: {}", table_)); + "System table can only contain one '$' separator, but this is: {}", object_)); } data_table = splits[0]; branch = splits[1].substr(std::strlen(kSystemBranchPrefix)); system_table = splits[2]; } else { - return Status::Invalid(fmt::format("Invalid table name: {}", table_)); + return Status::Invalid(fmt::format("Invalid object name: {}", object_)); } // A blank branch names the main branch everywhere else here, and the constructor above folds // it into the bare table name, so an object name spelling one was hand-built and would be // addressed as a branch of its own. if (data_table.empty() || (branch && StringUtils::IsNullOrWhitespaceOnly(branch.value())) || (system_table && system_table->empty())) { - return Status::Invalid(fmt::format("Invalid table name: {}", table_)); + return Status::Invalid(fmt::format("Invalid object name: {}", object_)); } data_table_ = std::move(data_table); branch_ = std::move(branch); diff --git a/src/paimon/core/catalog/identifier_test.cpp b/src/paimon/core/catalog/identifier_test.cpp index d358a9255..4b3c19f82 100644 --- a/src/paimon/core/catalog/identifier_test.cpp +++ b/src/paimon/core/catalog/identifier_test.cpp @@ -26,13 +26,7 @@ namespace paimon::test { TEST(IdentifierTest, ConstructorAndGetters) { Identifier id("test_db", "test_table"); ASSERT_EQ(id.GetDatabaseName(), "test_db"); - ASSERT_EQ(id.GetTableName(), "test_table"); -} - -TEST(IdentifierTest, SingleArgumentConstructorUsesUnknownDatabase) { - Identifier id("test_table"); - ASSERT_EQ(id.GetDatabaseName(), Identifier::kUnknownDatabase); - ASSERT_EQ(id.GetTableName(), "test_table"); + ASSERT_EQ(id.GetObjectName(), "test_table"); } TEST(IdentifierTest, EqualityOperator) { @@ -46,18 +40,18 @@ TEST(IdentifierTest, EqualityOperator) { TEST(IdentifierTest, ToString) { Identifier id("my_db", "my_table"); - ASSERT_EQ(id.ToString(), "Identifier{database='my_db', table='my_table'}"); + ASSERT_EQ(id.ToString(), "Identifier{database='my_db', object='my_table'}"); } TEST(IdentifierTest, EmptyDatabaseRemainsEmpty) { Identifier id("", "my_table"); ASSERT_EQ(id.GetDatabaseName(), ""); - ASSERT_EQ(id.GetTableName(), "my_table"); + ASSERT_EQ(id.GetObjectName(), "my_table"); } TEST(IdentifierTest, ParseDataTable) { Identifier id("db", "tbl"); - ASSERT_EQ(id.GetTableName(), "tbl"); + ASSERT_EQ(id.GetObjectName(), "tbl"); ASSERT_OK_AND_ASSIGN(std::string data_table_name, id.GetDataTableName()); ASSERT_EQ(data_table_name, "tbl"); ASSERT_OK_AND_ASSIGN(std::optional branch_name, id.GetBranchName()); @@ -72,7 +66,7 @@ TEST(IdentifierTest, ParseDataTable) { TEST(IdentifierTest, ParseSystemTable) { Identifier id("db", "tbl$options"); - ASSERT_EQ(id.GetTableName(), "tbl$options"); + ASSERT_EQ(id.GetObjectName(), "tbl$options"); ASSERT_OK_AND_ASSIGN(std::string data_table_name, id.GetDataTableName()); ASSERT_EQ(data_table_name, "tbl"); ASSERT_OK_AND_ASSIGN(std::optional branch_name, id.GetBranchName()); @@ -101,7 +95,7 @@ TEST(IdentifierTest, ParseBranchTable) { TEST(IdentifierTest, BranchConstructor) { Identifier branch_id("db", "tbl", "dev"); - ASSERT_EQ(branch_id.GetTableName(), "tbl$branch_dev"); + ASSERT_EQ(branch_id.GetObjectName(), "tbl$branch_dev"); ASSERT_OK_AND_ASSIGN(std::string data_table_name, branch_id.GetDataTableName()); ASSERT_EQ(data_table_name, "tbl"); ASSERT_OK_AND_ASSIGN(std::optional branch_name, branch_id.GetBranchName()); @@ -114,6 +108,25 @@ TEST(IdentifierTest, BranchConstructor) { ASSERT_EQ(Identifier("db", "tbl", " "), Identifier("db", "tbl")); } +TEST(IdentifierTest, MainBranchAliasFoldsInGetBranchNameOrDefault) { + // `GetBranchName` keeps the spelling the object name was written with, while + // `GetBranchNameOrDefault` folds any spelling of the main branch to `main`: the value + // a manager is bound to, which names no directory of its own. + for (const std::string& alias : {"MAIN", "Main", "mAiN"}) { + Identifier id("db", "tbl$branch_" + alias); + ASSERT_OK_AND_ASSIGN(std::optional branch_name, id.GetBranchName()); + ASSERT_TRUE(branch_name); + EXPECT_EQ(alias, branch_name.value()); + ASSERT_OK_AND_ASSIGN(std::string branch, id.GetBranchNameOrDefault()); + EXPECT_EQ(Identifier::kDefaultMainBranch, branch); + } + + // A branch of another name stays as written. + ASSERT_OK_AND_ASSIGN(std::string branch, + Identifier("db", "tbl$branch_dev").GetBranchNameOrDefault()); + EXPECT_EQ("dev", branch); +} + TEST(IdentifierTest, ParseBranchSystemTable) { Identifier id("db", "tbl$branch_dev$options"); ASSERT_OK_AND_ASSIGN(std::string data_table_name, id.GetDataTableName()); @@ -132,7 +145,7 @@ TEST(IdentifierTest, TestGetFullName) { { // test unknown database std::string table = "tbl$branch_dev$options"; - Identifier id(table); + Identifier id(Identifier::kUnknownDatabase, table); ASSERT_EQ(table, id.GetFullName()); } { @@ -175,15 +188,15 @@ TEST(IdentifierTest, InvalidSystemTableName) { "System table can only contain one '$' separator"); Identifier too_many("db", "tbl$branch_dev$options$extra"); - ASSERT_NOK_WITH_MSG(too_many.IsSystemTable(), "Invalid table name"); + ASSERT_NOK_WITH_MSG(too_many.IsSystemTable(), "Invalid object name"); } TEST(IdentifierTest, InvalidEmptySystemTableNameParts) { - ASSERT_NOK_WITH_MSG(Identifier("db", "$options").IsSystemTable(), "Invalid table name"); - ASSERT_NOK_WITH_MSG(Identifier("db", "tbl$").IsSystemTable(), "Invalid table name"); - ASSERT_NOK_WITH_MSG(Identifier("db", "tbl$branch_").IsSystemTable(), "Invalid table name"); - ASSERT_NOK_WITH_MSG(Identifier("db", "tbl$branch_ ").IsSystemTable(), "Invalid table name"); - ASSERT_NOK_WITH_MSG(Identifier("db", "tbl$branch_dev$").IsSystemTable(), "Invalid table name"); + ASSERT_NOK_WITH_MSG(Identifier("db", "$options").IsSystemTable(), "Invalid object name"); + ASSERT_NOK_WITH_MSG(Identifier("db", "tbl$").IsSystemTable(), "Invalid object name"); + ASSERT_NOK_WITH_MSG(Identifier("db", "tbl$branch_").IsSystemTable(), "Invalid object name"); + ASSERT_NOK_WITH_MSG(Identifier("db", "tbl$branch_ ").IsSystemTable(), "Invalid object name"); + ASSERT_NOK_WITH_MSG(Identifier("db", "tbl$branch_dev$").IsSystemTable(), "Invalid object name"); ASSERT_NOK_WITH_MSG(Identifier("db", "tbl$$options").IsSystemTable(), "System table can only contain one '$' separator"); } diff --git a/src/paimon/core/operation/file_store_commit_impl_test.cpp b/src/paimon/core/operation/file_store_commit_impl_test.cpp index 9600ce201..7f0e9032e 100644 --- a/src/paimon/core/operation/file_store_commit_impl_test.cpp +++ b/src/paimon/core/operation/file_store_commit_impl_test.cpp @@ -1105,7 +1105,7 @@ TEST_F(FileStoreCommitImplTest, TestCatalogCommitRetriesWhenRefused) { ASSERT_NE(failed.ToString().find(table_path_), std::string::npos) << failed.ToString(); ASSERT_NE(failed.ToString().find("through catalog"), std::string::npos) << failed.ToString(); - ASSERT_NE(failed.ToString().find("table='tbl'"), std::string::npos) << failed.ToString(); + ASSERT_NE(failed.ToString().find("object='tbl'"), std::string::npos) << failed.ToString(); ASSERT_NOK_WITH_MSG(failed, fmt::format("{} retries", max_retries)); ASSERT_EQ(catalog->CommitCalls().size(), static_cast(max_retries + 1)); @@ -1138,7 +1138,7 @@ TEST_F(FileStoreCommitImplTest, TestUnknownCatalogOutcomeKeepsTheManifests) { ASSERT_NOK_WITH_MSG(failed, "catalog unreachable"); ASSERT_NE(failed.ToString().find(table_path_), std::string::npos) << failed.ToString(); ASSERT_NE(failed.ToString().find("through catalog"), std::string::npos) << failed.ToString(); - ASSERT_NE(failed.ToString().find("table='tbl'"), std::string::npos) << failed.ToString(); + ASSERT_NE(failed.ToString().find("object='tbl'"), std::string::npos) << failed.ToString(); ASSERT_NE(failed.ToString().find("snapshot uuid none"), std::string::npos) << failed.ToString(); ASSERT_EQ(catalog->CommitCalls().size(), 1u); diff --git a/src/paimon/core/operation/read_context.cpp b/src/paimon/core/operation/read_context.cpp index 1779d7300..eb8710e0d 100644 --- a/src/paimon/core/operation/read_context.cpp +++ b/src/paimon/core/operation/read_context.cpp @@ -70,12 +70,10 @@ std::string IdentifierToFlatJson(const Identifier& identifier) { auto& allocator = document.GetAllocator(); document.AddMember(rapidjson::StringRef("database"), rapidjson::Value(identifier.GetDatabaseName(), allocator), allocator); - // C++ `GetTableName()` returns the stored object name - branch and system-table suffix - // included - which is the Java `getObjectName()` this field mirrors, not Java's stripped - // `getTableName()` (that one is `GetDataTableName()` here). So a branch read reports the - // branch. + // `GetObjectName()` keeps any branch and system-table suffix, matching the Java + // `getObjectName()` this field mirrors. So a branch read reports the branch. document.AddMember(rapidjson::StringRef("object"), - rapidjson::Value(identifier.GetTableName(), allocator), allocator); + rapidjson::Value(identifier.GetObjectName(), allocator), allocator); // Compact, not the pretty form `RapidJsonUtil::ToJson` writes: the value goes into a header. rapidjson::StringBuffer buffer; rapidjson::Writer writer(buffer); diff --git a/src/paimon/core/table/format/format_table_loader.cpp b/src/paimon/core/table/format/format_table_loader.cpp index c50b85e4f..797bc3351 100644 --- a/src/paimon/core/table/format/format_table_loader.cpp +++ b/src/paimon/core/table/format/format_table_loader.cpp @@ -78,7 +78,8 @@ Result> FormatTableLoader::TryLoad( } // A context names a path, never a catalog identifier, so the table is known by the directory // it sits in. - return FormatTable::Create(file_system, table_path, Identifier(PathUtil::GetName(table_path)), + Identifier identifier(Identifier::kUnknownDatabase, PathUtil::GetName(table_path)); + return FormatTable::Create(file_system, table_path, identifier, checked_pointer_cast(table_schema), /*location_carries_paimon_metadata=*/true, options); } diff --git a/src/paimon/core/table/system/system_table.cpp b/src/paimon/core/table/system/system_table.cpp index 2d67355a8..5a1a1de00 100644 --- a/src/paimon/core/table/system/system_table.cpp +++ b/src/paimon/core/table/system/system_table.cpp @@ -204,7 +204,8 @@ Result> SystemTableLoader::TryParsePath(const std return std::optional(std::move(system_table_path)); } - Identifier identifier(table_name); + // The database is unknown on a plain table path; only the object name is looked at below. + Identifier identifier(Identifier::kUnknownDatabase, table_name); PAIMON_ASSIGN_OR_RAISE(bool is_system_table, identifier.IsSystemTable()); if (!is_system_table) { return std::optional(); diff --git a/src/paimon/core/table/table.cpp b/src/paimon/core/table/table.cpp index c627b4581..f7123fc6e 100644 --- a/src/paimon/core/table/table.cpp +++ b/src/paimon/core/table/table.cpp @@ -7,29 +7,65 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ #include "paimon/catalog/table.h" +#include #include +#include +#include #include "fmt/format.h" +#include "paimon/commit_context.h" #include "paimon/common/utils/checked_cast.h" -#include "paimon/common/utils/string_utils.h" #include "paimon/core/catalog/catalog_utils.h" #include "paimon/core/schema/schema_manager.h" #include "paimon/core/schema/table_schema.h" +#include "paimon/core/utils/snapshot_manager.h" +#include "paimon/file_store_commit.h" +#include "paimon/file_store_write.h" #include "paimon/fs/file_system.h" +#include "paimon/read_context.h" +#include "paimon/scan_context.h" +#include "paimon/table/source/table_read.h" +#include "paimon/table/source/table_scan.h" +#include "paimon/write_context.h" namespace paimon { +Table::Table(const std::shared_ptr& file_system, const std::string& root_path, + const std::shared_ptr& schema, const Identifier& identifier, + const std::string& uuid, const std::shared_ptr& schema_manager) + : file_system_(file_system), + root_path_(root_path), + schema_(schema), + identifier_(identifier), + uuid_(uuid), + schema_manager_(schema_manager) { + // The managers are built on the table's own branch, which a malformed object name + // cannot name: such a table is handed out with none rather than fail here, since a + // constructor has no error to return. A null file system, which a mock in a test + // opens a table with, leaves none for the same reason. + Result branch = identifier_.GetBranchNameOrDefault(); + if (file_system_ == nullptr || !branch.ok()) { + schema_manager_ = nullptr; + return; + } + if (schema_manager_ == nullptr) { + schema_manager_ = std::make_shared(file_system_, root_path_, branch.value()); + } + snapshot_manager_ = std::make_shared(file_system_, root_path_, branch.value()); +} + Result> Table::Create(const std::shared_ptr& file_system, const std::string& table_path, const Identifier& identifier) { @@ -38,14 +74,16 @@ Result> Table::Create(const std::shared_ptr& return Status::NotExist(fmt::format("{} not exist", identifier.ToString())); } + // The schema is read through the manager the returned table keeps, so the file read + // here is the one read through the table as well. Built on the table's own branch: + // `root/branch/branch-` below the table path, which `main` resolves to itself. + // `GetBranchNameOrDefault` folds any spelling of `main`, so the managers the + // constructor keeps and `CurrentBranch()` agree with this one. PAIMON_RETURN_NOT_OK(CatalogUtils::CheckValidTableName(identifier)); PAIMON_ASSIGN_OR_RAISE(std::string branch, identifier.GetBranchNameOrDefault()); - if (StringUtils::EqualsIgnoreCase(branch, Identifier::kDefaultMainBranch)) { - branch = Identifier::kDefaultMainBranch; - } - SchemaManager schema_manager(file_system, table_path, branch); + auto schema_manager = std::make_shared(file_system, table_path, branch); PAIMON_ASSIGN_OR_RAISE(std::optional> latest_schema, - schema_manager.Latest()); + schema_manager->Latest()); if (!latest_schema) { return Status::NotExist( fmt::format("load table schema for {} failed", identifier.ToString())); @@ -58,11 +96,26 @@ Result> Table::Create(const std::shared_ptr& CatalogUtils::CheckManagedTableType(identifier, *latest_schema, "Table::Create")); auto schema = checked_pointer_cast(*latest_schema); - return std::make_shared
(schema, identifier.GetDatabaseName(), identifier.GetTableName()); + return std::make_shared
(file_system, table_path, schema, identifier, /*uuid=*/"", + schema_manager); +} + +Result> Table::NewScan(std::unique_ptr context) const { + return TableScan::Create(std::move(context)); +} + +Result> Table::NewRead(std::unique_ptr context) const { + return TableRead::Create(std::move(context)); +} + +Result> Table::NewWrite( + std::unique_ptr context) const { + return FileStoreWrite::Create(std::move(context)); } -std::string Table::FullName() const { - return database_ == Identifier::kUnknownDatabase ? table_name_ : database_ + "." + table_name_; +Result> Table::NewCommit( + std::unique_ptr context) const { + return FileStoreCommit::Create(std::move(context)); } } // namespace paimon diff --git a/src/paimon/core/table/table_test.cpp b/src/paimon/core/table/table_test.cpp index 8032890e6..6e1abf7c7 100644 --- a/src/paimon/core/table/table_test.cpp +++ b/src/paimon/core/table/table_test.cpp @@ -20,18 +20,49 @@ #include #include +#include #include #include #include "arrow/api.h" #include "gtest/gtest.h" +#include "paimon/commit_context.h" #include "paimon/core/schema/schema_manager.h" #include "paimon/core/schema/table_schema.h" +#include "paimon/core/snapshot.h" +#include "paimon/core/utils/snapshot_manager.h" +#include "paimon/file_store_commit.h" +#include "paimon/file_store_write.h" +#include "paimon/read_context.h" +#include "paimon/scan_context.h" #include "paimon/schema/schema.h" +#include "paimon/table/source/table_read.h" +#include "paimon/table/source/table_scan.h" +#include "paimon/testing/mock/mock_catalog.h" +#include "paimon/testing/utils/snapshot_test_helper.h" #include "paimon/testing/utils/testharness.h" +#include "paimon/write_context.h" namespace paimon::test { +namespace { + +// Creates a managed table directory and opens it as a `Table` under `identifier`. The +// schema goes to `branch`'s directory, which a branch identifier reads it back from. +Result> CreateManagedTable(const std::shared_ptr& file_system, + const std::string& path, + const Identifier& identifier, + const std::string& branch = "main") { + SchemaManager schema_manager(file_system, path, branch); + auto schema = arrow::schema({arrow::field("id", arrow::int32(), /*nullable=*/false)}); + PAIMON_ASSIGN_OR_RAISE([[maybe_unused]] std::unique_ptr created, + schema_manager.CreateTable(schema, /*partition_keys=*/{}, + /*primary_keys=*/{}, /*options=*/{})); + return Table::Create(file_system, path, identifier); +} + +} // namespace + TEST(TableTest, TestCreateWithUnknownDatabase) { auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); @@ -51,7 +82,8 @@ TEST(TableTest, TestCreateWithUnknownDatabase) { schema_manager.CreateTable(schema, partition_keys, primary_keys, options)); ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, - Table::Create(dir->GetFileSystem(), dir->Str(), Identifier("tbl1"))); + Table::Create(dir->GetFileSystem(), dir->Str(), + Identifier(Identifier::kUnknownDatabase, "tbl1"))); EXPECT_EQ(table->Name(), "tbl1"); EXPECT_EQ(table->FullName(), "tbl1"); @@ -96,6 +128,154 @@ TEST(TableTest, TestCreateWithBranch) { "branch"); } +TEST(TableTest, TestCreateHoldsTheFileSystem) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + const std::shared_ptr fs = dir->GetFileSystem(); + + ASSERT_OK_AND_ASSIGN( + std::shared_ptr
table, + CreateManagedTable(fs, dir->Str(), Identifier(Identifier::kUnknownDatabase, "tbl1"))); + + EXPECT_EQ(table->GetFileSystem(), fs); + EXPECT_EQ(table->GetRootPath(), dir->Str()); +} + +TEST(TableTest, TestManagersBoundToTheTable) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + const std::shared_ptr fs = dir->GetFileSystem(); + const Identifier identifier("db", "tbl"); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, + CreateManagedTable(fs, dir->Str(), identifier)); + + // The managers carry the table's own file system, root path and branch, which `main` + // names for an identifier that carries none. + std::shared_ptr snapshot_manager = table->GetSnapshotManager(); + ASSERT_TRUE(snapshot_manager != nullptr); + EXPECT_EQ(snapshot_manager->Fs(), fs); + EXPECT_EQ(snapshot_manager->RootPath(), dir->Str()); + EXPECT_EQ(snapshot_manager->Branch(), "main"); + + // The schema manager the table keeps is the one `Table::Create` read the schema + // through, so reading the latest schema again goes through the warmed cache. + std::shared_ptr schema_manager = table->GetSchemaManager(); + ASSERT_TRUE(schema_manager != nullptr); + ASSERT_OK_AND_ASSIGN(std::optional> latest_schema, + schema_manager->Latest()); + ASSERT_TRUE(latest_schema.has_value()); + EXPECT_EQ(latest_schema.value(), table->LatestSchema()); + + // A table of a branch keeps managers bound to that branch. + ASSERT_OK_AND_ASSIGN( + std::shared_ptr
branch_table, + CreateManagedTable(fs, dir->Str() + "/branch_tbl", Identifier("db", "tbl", "dev"), + /*branch=*/"dev")); + ASSERT_TRUE(branch_table->GetSchemaManager() != nullptr); + ASSERT_TRUE(branch_table->GetSnapshotManager() != nullptr); + EXPECT_EQ(branch_table->GetSnapshotManager()->Branch(), "dev"); +} + +TEST(TableTest, TestMainBranchAliasBindsTheManagersToTheMainBranch) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + auto fs = dir->GetFileSystem(); + SchemaManager schema_manager(fs, dir->Str()); + ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr schema, + schema_manager.CreateTable( + arrow::schema({arrow::field("c0", arrow::int32())}), {}, {}, {})); + // A snapshot published on the main branch, which a table opened through the alias + // reads back through the manager it keeps. + SnapshotManager snapshot_manager(fs, dir->Str()); + ASSERT_OK_AND_ASSIGN(std::string snapshot_json, BuildTestSnapshot(1).ToJsonString()); + ASSERT_OK(fs->WriteFile(snapshot_manager.SnapshotPath(1), snapshot_json, /*overwrite=*/false)); + + for (const auto& main_identifier : + {Identifier("db", "tbl"), Identifier("db", "tbl$branch_main"), + Identifier("db", "tbl$branch_MAIN")}) { + SCOPED_TRACE(main_identifier.ToString()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, + Table::Create(fs, dir->Str(), main_identifier)); + ASSERT_OK_AND_ASSIGN(std::string branch, table->CurrentBranch()); + EXPECT_EQ("main", branch); + // The managers are bound to the main branch the alias spells, whose snapshots and + // schemas live under the table root itself, so both are read back. + ASSERT_TRUE(table->GetSnapshotManager() != nullptr); + EXPECT_EQ("main", table->GetSnapshotManager()->Branch()); + ASSERT_OK_AND_ASSIGN(std::optional latest_id, + table->GetSnapshotManager()->LatestSnapshotId()); + ASSERT_TRUE(latest_id.has_value()); + EXPECT_EQ(1, latest_id.value()); + ASSERT_TRUE(table->GetSchemaManager() != nullptr); + ASSERT_OK_AND_ASSIGN(std::optional> latest_schema, + table->GetSchemaManager()->Latest()); + ASSERT_TRUE(latest_schema.has_value()); + EXPECT_EQ(latest_schema.value(), table->LatestSchema()); + } + + // A catalog that loaded the schema itself calls the constructor with no warmed schema + // manager, which builds its managers on its own; a rest catalog is shaped like this. + // The alias binds those to the main branch as well. + ASSERT_OK_AND_ASSIGN(std::shared_ptr
plain, + Table::Create(fs, dir->Str(), Identifier("db", "tbl"))); + Table alias_table(fs, dir->Str(), plain->LatestSchema(), Identifier("db", "tbl$branch_MAIN")); + ASSERT_OK_AND_ASSIGN(std::string alias_branch, alias_table.CurrentBranch()); + EXPECT_EQ("main", alias_branch); + ASSERT_OK_AND_ASSIGN(std::optional alias_latest_id, + alias_table.GetSnapshotManager()->LatestSnapshotId()); + ASSERT_TRUE(alias_latest_id.has_value()); + EXPECT_EQ(1, alias_latest_id.value()); + ASSERT_OK_AND_ASSIGN(std::optional> alias_latest_schema, + alias_table.GetSchemaManager()->Latest()); + ASSERT_TRUE(alias_latest_schema.has_value()); +} + +TEST(TableTest, TestCurrentBranch) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + + // A table whose identifier names no branch is a table of the main branch. + ASSERT_OK_AND_ASSIGN( + std::shared_ptr
main_table, + CreateManagedTable(dir->GetFileSystem(), dir->Str(), Identifier("db", "tbl"))); + ASSERT_OK_AND_ASSIGN(std::string branch, main_table->CurrentBranch()); + EXPECT_EQ(branch, "main"); + + // The identifier a table was opened with names the branch it is for. + ASSERT_OK_AND_ASSIGN(std::shared_ptr
branch_table, + CreateManagedTable(dir->GetFileSystem(), dir->Str() + "/branch_tbl", + Identifier("db", "tbl", "dev"), /*branch=*/"dev")); + ASSERT_OK_AND_ASSIGN(branch, branch_table->CurrentBranch()); + EXPECT_EQ(branch, "dev"); +} + +TEST(TableTest, TestCatalogOpenedTableServesThePerTableFileSystem) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + const Identifier identifier("db", "tbl"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, + CreateManagedTable(dir->GetFileSystem(), dir->Str(), identifier)); + + auto catalog = std::make_shared(); + catalog->SetTableSchema(table->LatestSchema()); + + // A catalog that hands out no per-table file system opens the table without one, and one + // that does opens it with the file system issued for that table. + ASSERT_OK_AND_ASSIGN(std::shared_ptr
opened, catalog->GetTable(identifier)); + EXPECT_EQ(opened->GetFileSystem(), nullptr); + // Without a file system the table keeps no managers either, which an operator has to + // build its own in place of. + EXPECT_EQ(opened->GetSchemaManager(), nullptr); + EXPECT_EQ(opened->GetSnapshotManager(), nullptr); + + catalog->SetTableFileSystem(dir->GetFileSystem()); + ASSERT_OK_AND_ASSIGN(opened, catalog->GetTable(identifier)); + EXPECT_EQ(opened->GetFileSystem(), dir->GetFileSystem()); + // This mock serves no table location, so the table it opens carries none. + EXPECT_EQ(opened->GetRootPath(), ""); +} + TEST(TableTest, TestCreateRejectsATableTypeThatIsNotAManagedTable) { // A `Table` describes a managed paimon table. Handing back another type as one would promise // snapshots and manifests it never had, so the failure belongs at the call that opens the @@ -148,8 +328,130 @@ TEST(TableTest, TestCreateFailedWithNonExistSchema) { ASSERT_OK(fs->Delete(schema_path, /*recursive=*/false)); // check create table failed ASSERT_NOK_WITH_MSG( - Table::Create(fs, dir->Str(), Identifier("tbl1")), - "load table schema for Identifier{database='unknown', table='tbl1'} failed"); + Table::Create(fs, dir->Str(), Identifier(Identifier::kUnknownDatabase, "tbl1")), + "load table schema for Identifier{database='unknown', object='tbl1'} failed"); +} + +TEST(TableTest, TestNewScanCreatesATableScan) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + const Identifier identifier("db", "tbl"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, + CreateManagedTable(dir->GetFileSystem(), dir->Str(), identifier)); + + auto catalog = std::make_shared(); + catalog->SetFileSystem(dir->GetFileSystem()); + + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, + ScanContextBuilder(dir->Str()).WithCatalog(catalog, identifier).Finish()); + ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr scan, + table->NewScan(std::move(ctx))); +} + +TEST(TableTest, TestNewReadCreatesATableRead) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + const Identifier identifier("db", "tbl"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, + CreateManagedTable(dir->GetFileSystem(), dir->Str(), identifier)); + + auto catalog = std::make_shared(); + catalog->SetFileSystem(dir->GetFileSystem()); + + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, + ReadContextBuilder(dir->Str()).WithCatalog(catalog, identifier).Finish()); + ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr read, + table->NewRead(std::move(ctx))); +} + +TEST(TableTest, TestNewWriteCreatesAFileStoreWrite) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + const Identifier identifier("db", "tbl"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, + CreateManagedTable(dir->GetFileSystem(), dir->Str(), identifier)); + + auto catalog = std::make_shared(); + catalog->SetFileSystem(dir->GetFileSystem()); + // A catalog write reads the current schema from the catalog on the main branch. + catalog->SetTableSchema(table->LatestSchema()); + + ASSERT_OK_AND_ASSIGN( + std::unique_ptr ctx, + WriteContextBuilder(dir->Str(), "commit-user").WithCatalog(catalog, identifier).Finish()); + ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr write, + table->NewWrite(std::move(ctx))); +} + +TEST(TableTest, TestNewCommitCreatesAFileStoreCommit) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + const Identifier identifier("db", "tbl"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, + CreateManagedTable(dir->GetFileSystem(), dir->Str(), identifier)); + + auto catalog = std::make_shared(); + catalog->SetFileSystem(dir->GetFileSystem()); + // Like a write, a catalog commit reads the current schema from the catalog. + catalog->SetTableSchema(table->LatestSchema()); + + ASSERT_OK_AND_ASSIGN( + std::unique_ptr ctx, + CommitContextBuilder(dir->Str(), "commit-user").WithCatalog(catalog, identifier).Finish()); + ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr commit, + table->NewCommit(std::move(ctx))); +} + +TEST(TableTest, TestNewScanReadWriteCommitRejectANullContext) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr
table, + CreateManagedTable(dir->GetFileSystem(), dir->Str(), Identifier("db", "tbl"))); + + // The operators report the null context themselves; the table stands in without swallowing + // the error, so behavior matches `TableScan::Create` and friends. + ASSERT_NOK_WITH_MSG(table->NewScan(nullptr), "scan context is null pointer"); + ASSERT_NOK_WITH_MSG(table->NewRead(nullptr), "read context is null pointer"); + ASSERT_NOK_WITH_MSG(table->NewWrite(nullptr), "write context is null pointer"); + ASSERT_NOK_WITH_MSG(table->NewCommit(nullptr), "commit context is null pointer"); +} + +TEST(TableTest, TestCreateBuildsTheOperator) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + const Identifier identifier("db", "tbl"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, + CreateManagedTable(dir->GetFileSystem(), dir->Str(), identifier)); + + auto catalog = std::make_shared(); + catalog->SetFileSystem(dir->GetFileSystem()); + // A catalog write and commit read the current schema from the catalog on the main branch. + catalog->SetTableSchema(table->LatestSchema()); + + // `TableScan::Create()` and friends build the operator straight from the context, which + // carries the catalog and identifier it resolves the table through. + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_ctx, + ScanContextBuilder(dir->Str()).WithCatalog(catalog, identifier).Finish()); + ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr scan, + TableScan::Create(std::move(scan_ctx))); + + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_ctx, + ReadContextBuilder(dir->Str()).WithCatalog(catalog, identifier).Finish()); + ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr read, + TableRead::Create(std::move(read_ctx))); + + ASSERT_OK_AND_ASSIGN( + std::unique_ptr write_ctx, + WriteContextBuilder(dir->Str(), "commit-user").WithCatalog(catalog, identifier).Finish()); + ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr write, + FileStoreWrite::Create(std::move(write_ctx))); + + ASSERT_OK_AND_ASSIGN( + std::unique_ptr commit_ctx, + CommitContextBuilder(dir->Str(), "commit-user").WithCatalog(catalog, identifier).Finish()); + ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr commit, + FileStoreCommit::Create(std::move(commit_ctx))); } } // namespace paimon::test diff --git a/src/paimon/core/utils/branch_manager_test.cpp b/src/paimon/core/utils/branch_manager_test.cpp index 307a53f4b..c73459241 100644 --- a/src/paimon/core/utils/branch_manager_test.cpp +++ b/src/paimon/core/utils/branch_manager_test.cpp @@ -90,7 +90,7 @@ TEST(BranchManagerTest, TestResolveBranch) { ASSERT_NOK_WITH_MSG(BranchManager::ResolveBranch(Identifier("db", "tbl$a$b$c"), no_options, std::nullopt, "commit"), - "Invalid table name"); + "Invalid object name"); ASSERT_NOK_WITH_MSG( BranchManager::ResolveBranch(Identifier("db", "tbl$branch_data"), diff --git a/src/paimon/rest/rest_api.cpp b/src/paimon/rest/rest_api.cpp index af5a78587..45865595d 100644 --- a/src/paimon/rest/rest_api.cpp +++ b/src/paimon/rest/rest_api.cpp @@ -273,11 +273,11 @@ Result> RestApi::ListTables(const std::string& database Result RestApi::GetTable(const Identifier& identifier) const { return GetEntity( - resource_paths_.Table(identifier.GetDatabaseName(), identifier.GetTableName()), {}); + resource_paths_.Table(identifier.GetDatabaseName(), identifier.GetObjectName()), {}); } Status RestApi::CreateTable(const Identifier& identifier, const std::string& schema_json) const { - CreateTableRequest request(identifier.GetDatabaseName(), identifier.GetTableName(), + CreateTableRequest request(identifier.GetDatabaseName(), identifier.GetObjectName(), schema_json); PAIMON_ASSIGN_OR_RAISE(std::string body, request.ToJsonString()); return Execute("POST", resource_paths_.Tables(identifier.GetDatabaseName()), {}, body).status(); @@ -285,26 +285,26 @@ Status RestApi::CreateTable(const Identifier& identifier, const std::string& sch Status RestApi::DropTable(const Identifier& identifier) const { return Execute("DELETE", - resource_paths_.Table(identifier.GetDatabaseName(), identifier.GetTableName()), + resource_paths_.Table(identifier.GetDatabaseName(), identifier.GetObjectName()), {}, "") .status(); } Status RestApi::RenameTable(const Identifier& from_table, const Identifier& to_table) const { - RenameTableRequest request(from_table.GetDatabaseName(), from_table.GetTableName(), - to_table.GetDatabaseName(), to_table.GetTableName()); + RenameTableRequest request(from_table.GetDatabaseName(), from_table.GetObjectName(), + to_table.GetDatabaseName(), to_table.GetObjectName()); PAIMON_ASSIGN_OR_RAISE(std::string body, request.ToJsonString()); return Execute("POST", resource_paths_.RenameTable(), {}, body).status(); } Result> RestApi::ListSnapshots(const Identifier& identifier) const { return ListAllPages( - resource_paths_.Snapshots(identifier.GetDatabaseName(), identifier.GetTableName())); + resource_paths_.Snapshots(identifier.GetDatabaseName(), identifier.GetObjectName())); } Result> RestApi::LoadSnapshot(const Identifier& identifier) const { std::string path = - resource_paths_.TableSnapshot(identifier.GetDatabaseName(), identifier.GetTableName()); + resource_paths_.TableSnapshot(identifier.GetDatabaseName(), identifier.GetObjectName()); Result response = GetEntity(path, /*query_params=*/{}); if (response.ok()) { @@ -329,7 +329,7 @@ Result RestApi::CommitSnapshot(const Identifier& identifier, CommitTableRequest request(table_uuid, base_snapshot_uuid, snapshot, statistics); PAIMON_ASSIGN_OR_RAISE(std::string body, request.ToJsonString()); std::string path = - resource_paths_.CommitTable(identifier.GetDatabaseName(), identifier.GetTableName()); + resource_paths_.CommitTable(identifier.GetDatabaseName(), identifier.GetObjectName()); // Replaying an accepted commit can report a conflict and trigger manifest deletion. // Send once and let FilterAndCommit() recover an unknown outcome. PAIMON_ASSIGN_OR_RAISE(RestHttpClient::Response response, @@ -341,7 +341,7 @@ Result RestApi::CommitSnapshot(const Identifier& identifier, Result RestApi::LoadTableToken(const Identifier& identifier) const { return GetEntity( - resource_paths_.TableToken(identifier.GetDatabaseName(), identifier.GetTableName()), {}); + resource_paths_.TableToken(identifier.GetDatabaseName(), identifier.GetObjectName()), {}); } } // namespace paimon diff --git a/src/paimon/rest/rest_catalog.cpp b/src/paimon/rest/rest_catalog.cpp index acbc0cb78..66898d8af 100644 --- a/src/paimon/rest/rest_catalog.cpp +++ b/src/paimon/rest/rest_catalog.cpp @@ -230,7 +230,7 @@ Status RestCatalog::CreateTable(const Identifier& identifier, ArrowSchema* c_sch Result RestCatalog::TableExists(const Identifier& identifier) const { if (CatalogUtils::IsSystemDatabase(identifier.GetDatabaseName())) { - return GlobalSystemTableLoader::IsSupported(identifier.GetTableName(), + return GlobalSystemTableLoader::IsSupported(identifier.GetObjectName(), api_->GetMergedOptions()); } PAIMON_ASSIGN_OR_RAISE(bool is_system_table, identifier.IsSystemTable()); @@ -366,16 +366,17 @@ Result> RestCatalog::LoadDataTableSchema( } Result> RestCatalog::LoadTableSchema(const Identifier& identifier) const { - return LoadTableSchema(identifier, /*table_id=*/nullptr); + return LoadTableSchema(identifier, /*table_id=*/nullptr, /*table_path=*/nullptr); } Result> RestCatalog::LoadTableSchema(const Identifier& identifier, - std::string* table_id) const { + std::string* table_id, + std::string* table_path) const { // Serve the global system tables of the "sys" database locally, like // FileSystemCatalog. if (CatalogUtils::IsSystemDatabase(identifier.GetDatabaseName())) { PAIMON_ASSIGN_OR_RAISE(bool supported, - GlobalSystemTableLoader::IsSupported(identifier.GetTableName(), + GlobalSystemTableLoader::IsSupported(identifier.GetObjectName(), api_->GetMergedOptions())); if (!supported) { return Status::NotExist(fmt::format("{} not exist", identifier.ToString())); @@ -386,7 +387,7 @@ Result> RestCatalog::LoadTableSchema(const Identifier& i context.warehouse = warehouse_; context.catalog_options = api_->GetMergedOptions(); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr system_table, - GlobalSystemTableLoader::Load(identifier.GetTableName(), context)); + GlobalSystemTableLoader::Load(identifier.GetObjectName(), context)); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr arrow_schema, system_table->ArrowSchema()); return std::make_shared(std::move(arrow_schema)); @@ -401,10 +402,13 @@ Result> RestCatalog::LoadTableSchema(const Identifier& i if (!system_table_name || !SystemTableLoader::IsSupported(system_table_name.value())) { return Status::NotExist(fmt::format("{} not exist", identifier.ToString())); } - std::string table_path; + std::string data_table_path; PAIMON_ASSIGN_OR_RAISE(std::shared_ptr latest_schema, - LoadDataTableSchema(load_identifier, branch, &table_path, + LoadDataTableSchema(load_identifier, branch, &data_table_path, /*table_id=*/nullptr)); + if (table_path != nullptr) { + *table_path = data_table_path; + } std::map dynamic_options; if (branch) { dynamic_options[Options::BRANCH] = branch.value(); @@ -413,15 +417,16 @@ Result> RestCatalog::LoadTableSchema(const Identifier& i // read any file, so the catalog wide one is enough here. The credentials of the // table are applied when its rows are read, through the file system of the read or // scan context. - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr system_table, - SystemTableLoader::Load(system_table_name.value(), fs_, table_path, - latest_schema, dynamic_options)); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr system_table, + SystemTableLoader::Load(system_table_name.value(), fs_, data_table_path, latest_schema, + dynamic_options)); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr arrow_schema, system_table->ArrowSchema()); return std::make_shared(std::move(arrow_schema)); } PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, - LoadDataTableSchema(load_identifier, branch, nullptr, table_id)); + LoadDataTableSchema(load_identifier, branch, table_path, table_id)); return checked_pointer_cast(schema); } @@ -443,13 +448,18 @@ Result> RestCatalog::LoadFormatTable( } Result> RestCatalog::GetTable(const Identifier& identifier) const { - // Read UUID and schema together to avoid mixing table generations after recreation. + // Read UUID, path and schema together to avoid mixing table generations after recreation. std::string uuid; - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, LoadTableSchema(identifier, &uuid)); + std::string table_path; + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, + LoadTableSchema(identifier, &uuid, &table_path)); PAIMON_RETURN_NOT_OK( CatalogUtils::CheckManagedTableType(identifier, schema, "Catalog::GetTable")); - return std::make_shared
(schema, identifier.GetDatabaseName(), identifier.GetTableName(), - uuid); + // The table holds the file system its own data is served by, which a server issuing data + // tokens binds to this table. + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table_file_system, + GetTableFileSystem(identifier)); + return std::make_shared
(table_file_system, table_path, schema, identifier, uuid); } std::string RestCatalog::GetRootPath() const { @@ -494,7 +504,7 @@ Result> RestCatalog::ListSnapshots(const Identifier& i PAIMON_RETURN_NOT_OK(CatalogUtils::CheckNotSystemTable(identifier, "listSnapshots")); // `CheckNotBranch` and `CheckNotSystemTable` above leave the table name bare, so the branch // this is asked for is the only one the object name can carry. - Identifier load_identifier(identifier.GetDatabaseName(), identifier.GetTableName(), branch); + Identifier load_identifier(identifier.GetDatabaseName(), identifier.GetObjectName(), branch); // The Catalog interface has no pagination, so all pages are fetched; the server // does not order snapshots across pages while the contract requires ascending ids. PAIMON_ASSIGN_OR_RAISE(std::vector snapshots, api_->ListSnapshots(load_identifier)); @@ -512,7 +522,7 @@ Result> RestCatalog::GetLatestSnapshot( const Identifier& identifier, const std::string& branch) const { PAIMON_RETURN_NOT_OK(CatalogUtils::CheckNotBranch(identifier, "getLatestSnapshot")); PAIMON_RETURN_NOT_OK(CatalogUtils::CheckNotSystemTable(identifier, "getLatestSnapshot")); - Identifier load_identifier(identifier.GetDatabaseName(), identifier.GetTableName(), branch); + Identifier load_identifier(identifier.GetDatabaseName(), identifier.GetObjectName(), branch); PAIMON_ASSIGN_OR_RAISE(std::optional snapshot, api_->LoadSnapshot(load_identifier)); if (!snapshot) { return std::optional(); diff --git a/src/paimon/rest/rest_catalog.h b/src/paimon/rest/rest_catalog.h index 2659b67c4..051b90d38 100644 --- a/src/paimon/rest/rest_catalog.h +++ b/src/paimon/rest/rest_catalog.h @@ -122,8 +122,10 @@ class RestCatalog : public Catalog, public VersionManagedCatalog { const std::map& fs_scheme_to_identifier_map); /// Loads the schema and catalog table ID from the same response. + /// Fills `table_path` with the table location when supplied. Result> LoadTableSchema(const Identifier& identifier, - std::string* table_id) const; + std::string* table_id, + std::string* table_path) const; /// Loads the schema with table path, audit and branch options. /// Fills `table_path` and `table_id` when supplied. diff --git a/src/paimon/rest/rest_catalog_test.cpp b/src/paimon/rest/rest_catalog_test.cpp index 90a8a48c3..968da94a2 100644 --- a/src/paimon/rest/rest_catalog_test.cpp +++ b/src/paimon/rest/rest_catalog_test.cpp @@ -44,6 +44,7 @@ #include "paimon/core/catalog/commit_table_request.h" #include "paimon/core/partition/partition_statistics.h" #include "paimon/core/schema/table_schema.h" +#include "paimon/core/utils/snapshot_manager.h" #include "paimon/defs.h" #include "paimon/file_store_commit.h" #include "paimon/fs/file_system.h" @@ -1064,6 +1065,11 @@ TEST_F(RestCatalogTest, BranchTableLoadsBranchSchemaFromServer) { ASSERT_NE(nullptr, main_case_schema); ASSERT_EQ((std::vector{"f0", "f1"}), main_case_schema->FieldNames()); ASSERT_EQ(0, main_case_schema->Options().count(Options::BRANCH)); + // the managers the table keeps are bound to the main branch the alias spells, not to + // a branch directory named after the alias + ASSERT_OK_AND_ASSIGN(std::string main_case_branch, main_case_table->CurrentBranch()); + ASSERT_EQ("main", main_case_branch); + ASSERT_EQ("main", main_case_table->GetSnapshotManager()->Branch()); // a system table on a branch resolves against the branch's data table: the system // suffix is stripped while the branch stays in the identifier sent to the server diff --git a/src/paimon/testing/mock/mock_catalog.h b/src/paimon/testing/mock/mock_catalog.h index b9954e313..67449abf2 100644 --- a/src/paimon/testing/mock/mock_catalog.h +++ b/src/paimon/testing/mock/mock_catalog.h @@ -95,8 +95,12 @@ class MockVersionManagedCatalog : public Catalog, public VersionManagedCatalog { Result> GetTable(const Identifier& identifier) const override { ++get_table_calls_; - return std::make_shared
(table_schema_, identifier.GetDatabaseName(), - identifier.GetTableName(), table_uuid_); + // Serves the per-table file system when one was set, matching `GetTableFileSystem` + // without recording the request, so a test can still tell the two apart. No location: + // this mock's `GetTableLocation` is the unsupported stub. + return std::make_shared
( + table_file_system_ != nullptr ? table_file_system_ : file_system_, + /*root_path=*/"", table_schema_, identifier, table_uuid_); } void SetTableSchema(const std::shared_ptr& table_schema) { From bab6771612c8af9247d9972ecc88b71bddf87f7d Mon Sep 17 00:00:00 2001 From: "yonghao.fyh" Date: Mon, 19 Oct 2026 14:38:08 +0800 Subject: [PATCH 2/6] fix --- include/paimon/catalog/table.h | 24 +----- include/paimon/read_context.h | 16 +++- include/paimon/scan_context.h | 16 +++- include/paimon/type_fwd.h | 3 - src/paimon/core/operation/read_context.cpp | 10 ++- .../core/operation/read_context_test.cpp | 7 +- src/paimon/core/operation/scan_context.cpp | 29 +++---- .../core/operation/scan_context_test.cpp | 7 +- src/paimon/core/table/table.cpp | 35 ++------- src/paimon/core/table/table_test.cpp | 75 +------------------ src/paimon/rest/rest_catalog_test.cpp | 5 +- 11 files changed, 78 insertions(+), 149 deletions(-) diff --git a/include/paimon/catalog/table.h b/include/paimon/catalog/table.h index 2194355c8..88a7e9b9e 100644 --- a/include/paimon/catalog/table.h +++ b/include/paimon/catalog/table.h @@ -52,12 +52,9 @@ class PAIMON_EXPORT Table { /// @param file_system File system the table's data is accessed through. /// @param root_path Root path of the table in the file system. /// @param uuid Metastore UUID, or empty if unavailable. - /// @param schema_manager A schema manager the table keeps instead of building its own, - /// holding the schema cache an opener already warmed; built when null. Table(const std::shared_ptr& file_system, const std::string& root_path, const std::shared_ptr& schema, const Identifier& identifier, - const std::string& uuid = "", - const std::shared_ptr& schema_manager = nullptr); + const std::string& uuid = ""); ~Table() = default; @@ -74,23 +71,6 @@ class PAIMON_EXPORT Table { return root_path_; } - /// Returns the manager of the table's schema versions, bound to the table's file - /// system, root path and branch, so schema files read through one table are not - /// read again through another. It is null when the table was opened without a - /// file system or with an identifier whose branch cannot be parsed, which a mock - /// in a test is one case of. - std::shared_ptr GetSchemaManager() const { - return schema_manager_; - } - - /// Returns the manager of the table's snapshots, bound like the schema manager. - /// It carries no snapshot loader: an operator that needs one attaches it to a - /// manager of its own, since a loader attached here would leak into every read - /// through this table. - std::shared_ptr GetSnapshotManager() const { - return snapshot_manager_; - } - /// A name to identify this table. std::string Name() const { return identifier_.GetObjectName(); @@ -157,8 +137,6 @@ class PAIMON_EXPORT Table { std::shared_ptr schema_; Identifier identifier_; std::string uuid_; - std::shared_ptr schema_manager_; - std::shared_ptr snapshot_manager_; }; } // namespace paimon diff --git a/include/paimon/read_context.h b/include/paimon/read_context.h index d9b37516a..716110c5e 100644 --- a/include/paimon/read_context.h +++ b/include/paimon/read_context.h @@ -69,7 +69,9 @@ class PAIMON_EXPORT ReadContext { const std::shared_ptr& realtime_context, const std::map& options, bool read_ahead_cache_enabled, const CacheConfig& cache_config, const std::shared_ptr& cache, - const std::shared_ptr& format_table, WarmupLevel warmup_level); + const std::shared_ptr& format_table, WarmupLevel warmup_level, + const std::shared_ptr& catalog = nullptr, + const std::optional& identifier = std::nullopt); ~ReadContext(); @@ -174,6 +176,16 @@ class PAIMON_EXPORT ReadContext { return warmup_level_; } + /// Returns the catalog supplying table metadata, or null if unset. + const std::shared_ptr& GetCatalog() const { + return catalog_; + } + + /// Returns the table identifier supplied with the catalog. + const std::optional& GetIdentifier() const { + return identifier_; + } + /// Whether a read schema (C ArrowSchema) for nested column pruning was provided. bool HasReadSchema() const { return read_schema_ != nullptr && read_schema_->release != nullptr; @@ -217,6 +229,8 @@ class PAIMON_EXPORT ReadContext { std::shared_ptr cache_; std::shared_ptr format_table_; WarmupLevel warmup_level_; + std::shared_ptr catalog_; + std::optional identifier_; // Owns schema resources and releases ArrowSchema::release in destructor. std::unique_ptr read_schema_; }; diff --git a/include/paimon/scan_context.h b/include/paimon/scan_context.h index 8f87d1210..fac2a1a35 100644 --- a/include/paimon/scan_context.h +++ b/include/paimon/scan_context.h @@ -58,7 +58,9 @@ class PAIMON_EXPORT ScanContext { const std::optional& table_schema, const std::map& options, const std::shared_ptr& cache, - const std::shared_ptr& format_table); + const std::shared_ptr& format_table, + const std::shared_ptr& catalog = nullptr, + const std::optional& identifier = std::nullopt); ~ScanContext(); @@ -115,6 +117,16 @@ class PAIMON_EXPORT ScanContext { return format_table_; } + /// Returns the catalog supplying table metadata, or null if unset. + const std::shared_ptr& GetCatalog() const { + return catalog_; + } + + /// Returns the table identifier supplied with the catalog. + const std::optional& GetIdentifier() const { + return identifier_; + } + private: std::string path_; bool is_streaming_mode_; @@ -129,6 +141,8 @@ class PAIMON_EXPORT ScanContext { std::map options_; std::shared_ptr cache_; std::shared_ptr format_table_; + std::shared_ptr catalog_; + std::optional identifier_; }; /// Filter configuration for table scan operations diff --git a/include/paimon/type_fwd.h b/include/paimon/type_fwd.h index f2c93a0bb..77efb108f 100644 --- a/include/paimon/type_fwd.h +++ b/include/paimon/type_fwd.h @@ -75,9 +75,6 @@ class FileStoreCommit; class WriteContext; class CommitContext; -class SchemaManager; -class SnapshotManager; - class CommitMessage; class Metrics; diff --git a/src/paimon/core/operation/read_context.cpp b/src/paimon/core/operation/read_context.cpp index eb8710e0d..8389ed909 100644 --- a/src/paimon/core/operation/read_context.cpp +++ b/src/paimon/core/operation/read_context.cpp @@ -97,7 +97,8 @@ ReadContext::ReadContext( const std::shared_ptr& realtime_context, const std::map& options, bool read_ahead_cache_enabled, const CacheConfig& cache_config, const std::shared_ptr& cache, - const std::shared_ptr& format_table, WarmupLevel warmup_level) + const std::shared_ptr& format_table, WarmupLevel warmup_level, + const std::shared_ptr& catalog, const std::optional& identifier) : path_(path), branch_(branch), read_field_names_(read_field_names), @@ -123,7 +124,9 @@ ReadContext::ReadContext( cache_config_(cache_config), cache_(cache), format_table_(format_table), - warmup_level_(warmup_level) {} + warmup_level_(warmup_level), + catalog_(catalog), + identifier_(identifier) {} ReadContext::~ReadContext() { if (read_schema_ && read_schema_->release) { @@ -478,7 +481,8 @@ Result> ReadContextBuilder::Finish() { impl_->row_to_batch_thread_number_, impl_->table_schema_, impl_->memory_pool_, impl_->executor_, impl_->specific_file_system_, impl_->fs_scheme_to_identifier_map_, impl_->realtime_context_, impl_->options_, impl_->read_ahead_cache_enabled_, - impl_->cache_config_, impl_->cache_, impl_->format_table_, impl_->warmup_level_); + impl_->cache_config_, impl_->cache_, impl_->format_table_, impl_->warmup_level_, + impl_->catalog_, impl_->identifier_); if (impl_->read_schema_ && impl_->read_schema_->release) { ctx->SetReadSchema(std::move(impl_->read_schema_)); } diff --git a/src/paimon/core/operation/read_context_test.cpp b/src/paimon/core/operation/read_context_test.cpp index 176df8b0e..35955807d 100644 --- a/src/paimon/core/operation/read_context_test.cpp +++ b/src/paimon/core/operation/read_context_test.cpp @@ -40,7 +40,8 @@ namespace paimon::test { TEST(ReadContextTest, TestWithCatalogResolvesTableFileSystem) { // WithCatalog is a shorthand for WithFileSystem(catalog->GetTableFileSystem(identifier)); it - // resolves the per-table file system when Finish() builds the context and sets nothing else. + // resolves the per-table file system when Finish() builds the context, which also keeps the + // catalog and identifier for the operators to resolve the table through. auto table_fs = std::make_shared(); auto catalog = std::make_shared(); catalog->SetTableFileSystem(table_fs); @@ -48,12 +49,16 @@ TEST(ReadContextTest, TestWithCatalogResolvesTableFileSystem) { ReadContextBuilder builder("table_root_path"); ASSERT_OK_AND_ASSIGN(auto ctx, builder.WithCatalog(catalog, Identifier("db1", "t1")).Finish()); ASSERT_EQ(ctx->GetSpecificFileSystem(), table_fs); + ASSERT_EQ(ctx->GetCatalog(), catalog); + ASSERT_EQ(ctx->GetIdentifier(), std::optional(Identifier("db1", "t1"))); ASSERT_EQ(catalog->TableFileSystemRequests().size(), 1U); ASSERT_EQ(catalog->TableFileSystemRequests().front(), Identifier("db1", "t1")); // Finish() resets the builder, so the next context reads without the catalog. ASSERT_OK_AND_ASSIGN(auto next_ctx, builder.Finish()); ASSERT_FALSE(next_ctx->GetSpecificFileSystem()); + ASSERT_FALSE(next_ctx->GetCatalog()); + ASSERT_FALSE(next_ctx->GetIdentifier()); ASSERT_EQ(catalog->TableFileSystemRequests().size(), 1U); } diff --git a/src/paimon/core/operation/scan_context.cpp b/src/paimon/core/operation/scan_context.cpp index bdef97321..89cccaa69 100644 --- a/src/paimon/core/operation/scan_context.cpp +++ b/src/paimon/core/operation/scan_context.cpp @@ -30,18 +30,17 @@ namespace paimon { class Predicate; -ScanContext::ScanContext(const std::string& path, bool is_streaming_mode, - std::optional limit, - const std::shared_ptr& scan_filter, - const std::shared_ptr& global_index_result, - const std::shared_ptr& realtime_context, - const std::shared_ptr& memory_pool, - const std::shared_ptr& executor, - const std::shared_ptr& specific_file_system, - const std::optional& table_schema, - const std::map& options, - const std::shared_ptr& cache, - const std::shared_ptr& format_table) +ScanContext::ScanContext( + const std::string& path, bool is_streaming_mode, std::optional limit, + const std::shared_ptr& scan_filter, + const std::shared_ptr& global_index_result, + const std::shared_ptr& realtime_context, + const std::shared_ptr& memory_pool, const std::shared_ptr& executor, + const std::shared_ptr& specific_file_system, + const std::optional& table_schema, + const std::map& options, const std::shared_ptr& cache, + const std::shared_ptr& format_table, const std::shared_ptr& catalog, + const std::optional& identifier) : path_(path), is_streaming_mode_(is_streaming_mode), limit_(limit), @@ -54,7 +53,9 @@ ScanContext::ScanContext(const std::string& path, bool is_streaming_mode, table_schema_(table_schema), options_(options), cache_(cache), - format_table_(format_table) {} + format_table_(format_table), + catalog_(catalog), + identifier_(identifier) {} ScanContext::~ScanContext() = default; @@ -247,7 +248,7 @@ Result> ScanContextBuilder::Finish() { impl_->bucket_filter_), impl_->global_index_result_, impl_->realtime_context_, impl_->memory_pool_, executor, impl_->specific_file_system_, impl_->table_schema_, impl_->options_, impl_->cache_, - impl_->format_table_); + impl_->format_table_, impl_->catalog_, impl_->identifier_); impl_->Reset(); return ctx; } diff --git a/src/paimon/core/operation/scan_context_test.cpp b/src/paimon/core/operation/scan_context_test.cpp index bc08b841f..4ce2395b3 100644 --- a/src/paimon/core/operation/scan_context_test.cpp +++ b/src/paimon/core/operation/scan_context_test.cpp @@ -33,7 +33,8 @@ namespace paimon::test { TEST(ScanContextTest, TestWithCatalogResolvesTableFileSystem) { // WithCatalog is a shorthand for WithFileSystem(catalog->GetTableFileSystem(identifier)); it - // resolves the per-table file system when Finish() builds the context and sets nothing else. + // resolves the per-table file system when Finish() builds the context, which also keeps the + // catalog and identifier for the operators to resolve the table through. auto table_fs = std::make_shared(); auto catalog = std::make_shared(); catalog->SetTableFileSystem(table_fs); @@ -41,12 +42,16 @@ TEST(ScanContextTest, TestWithCatalogResolvesTableFileSystem) { ScanContextBuilder builder("table_root_path"); ASSERT_OK_AND_ASSIGN(auto ctx, builder.WithCatalog(catalog, Identifier("db1", "t1")).Finish()); ASSERT_EQ(ctx->GetSpecificFileSystem(), table_fs); + ASSERT_EQ(ctx->GetCatalog(), catalog); + ASSERT_EQ(ctx->GetIdentifier(), std::optional(Identifier("db1", "t1"))); ASSERT_EQ(catalog->TableFileSystemRequests().size(), 1U); ASSERT_EQ(catalog->TableFileSystemRequests().front(), Identifier("db1", "t1")); // Finish() resets the builder, so the next context scans without the catalog. ASSERT_OK_AND_ASSIGN(auto next_ctx, builder.Finish()); ASSERT_FALSE(next_ctx->GetSpecificFileSystem()); + ASSERT_FALSE(next_ctx->GetCatalog()); + ASSERT_FALSE(next_ctx->GetIdentifier()); ASSERT_EQ(catalog->TableFileSystemRequests().size(), 1U); } diff --git a/src/paimon/core/table/table.cpp b/src/paimon/core/table/table.cpp index f7123fc6e..bc1e64fbc 100644 --- a/src/paimon/core/table/table.cpp +++ b/src/paimon/core/table/table.cpp @@ -30,7 +30,6 @@ #include "paimon/core/catalog/catalog_utils.h" #include "paimon/core/schema/schema_manager.h" #include "paimon/core/schema/table_schema.h" -#include "paimon/core/utils/snapshot_manager.h" #include "paimon/file_store_commit.h" #include "paimon/file_store_write.h" #include "paimon/fs/file_system.h" @@ -44,27 +43,12 @@ namespace paimon { Table::Table(const std::shared_ptr& file_system, const std::string& root_path, const std::shared_ptr& schema, const Identifier& identifier, - const std::string& uuid, const std::shared_ptr& schema_manager) + const std::string& uuid) : file_system_(file_system), root_path_(root_path), schema_(schema), identifier_(identifier), - uuid_(uuid), - schema_manager_(schema_manager) { - // The managers are built on the table's own branch, which a malformed object name - // cannot name: such a table is handed out with none rather than fail here, since a - // constructor has no error to return. A null file system, which a mock in a test - // opens a table with, leaves none for the same reason. - Result branch = identifier_.GetBranchNameOrDefault(); - if (file_system_ == nullptr || !branch.ok()) { - schema_manager_ = nullptr; - return; - } - if (schema_manager_ == nullptr) { - schema_manager_ = std::make_shared(file_system_, root_path_, branch.value()); - } - snapshot_manager_ = std::make_shared(file_system_, root_path_, branch.value()); -} + uuid_(uuid) {} Result> Table::Create(const std::shared_ptr& file_system, const std::string& table_path, @@ -74,16 +58,14 @@ Result> Table::Create(const std::shared_ptr& return Status::NotExist(fmt::format("{} not exist", identifier.ToString())); } - // The schema is read through the manager the returned table keeps, so the file read - // here is the one read through the table as well. Built on the table's own branch: - // `root/branch/branch-` below the table path, which `main` resolves to itself. - // `GetBranchNameOrDefault` folds any spelling of `main`, so the managers the - // constructor keeps and `CurrentBranch()` agree with this one. + // The schema is read on the table's own branch: `root/branch/branch-` below + // the table path, which `main` resolves to itself. `GetBranchNameOrDefault` folds + // any spelling of `main`, so the branch read here agrees with `CurrentBranch()`. PAIMON_RETURN_NOT_OK(CatalogUtils::CheckValidTableName(identifier)); PAIMON_ASSIGN_OR_RAISE(std::string branch, identifier.GetBranchNameOrDefault()); - auto schema_manager = std::make_shared(file_system, table_path, branch); + SchemaManager schema_manager(file_system, table_path, branch); PAIMON_ASSIGN_OR_RAISE(std::optional> latest_schema, - schema_manager->Latest()); + schema_manager.Latest()); if (!latest_schema) { return Status::NotExist( fmt::format("load table schema for {} failed", identifier.ToString())); @@ -96,8 +78,7 @@ Result> Table::Create(const std::shared_ptr& CatalogUtils::CheckManagedTableType(identifier, *latest_schema, "Table::Create")); auto schema = checked_pointer_cast(*latest_schema); - return std::make_shared
(file_system, table_path, schema, identifier, /*uuid=*/"", - schema_manager); + return std::make_shared
(file_system, table_path, schema, identifier, /*uuid=*/""); } Result> Table::NewScan(std::unique_ptr context) const { diff --git a/src/paimon/core/table/table_test.cpp b/src/paimon/core/table/table_test.cpp index 6e1abf7c7..da0cae11d 100644 --- a/src/paimon/core/table/table_test.cpp +++ b/src/paimon/core/table/table_test.cpp @@ -29,8 +29,6 @@ #include "paimon/commit_context.h" #include "paimon/core/schema/schema_manager.h" #include "paimon/core/schema/table_schema.h" -#include "paimon/core/snapshot.h" -#include "paimon/core/utils/snapshot_manager.h" #include "paimon/file_store_commit.h" #include "paimon/file_store_write.h" #include "paimon/read_context.h" @@ -39,7 +37,6 @@ #include "paimon/table/source/table_read.h" #include "paimon/table/source/table_scan.h" #include "paimon/testing/mock/mock_catalog.h" -#include "paimon/testing/utils/snapshot_test_helper.h" #include "paimon/testing/utils/testharness.h" #include "paimon/write_context.h" @@ -141,43 +138,7 @@ TEST(TableTest, TestCreateHoldsTheFileSystem) { EXPECT_EQ(table->GetRootPath(), dir->Str()); } -TEST(TableTest, TestManagersBoundToTheTable) { - auto dir = UniqueTestDirectory::Create(); - ASSERT_TRUE(dir); - const std::shared_ptr fs = dir->GetFileSystem(); - const Identifier identifier("db", "tbl"); - - ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, - CreateManagedTable(fs, dir->Str(), identifier)); - - // The managers carry the table's own file system, root path and branch, which `main` - // names for an identifier that carries none. - std::shared_ptr snapshot_manager = table->GetSnapshotManager(); - ASSERT_TRUE(snapshot_manager != nullptr); - EXPECT_EQ(snapshot_manager->Fs(), fs); - EXPECT_EQ(snapshot_manager->RootPath(), dir->Str()); - EXPECT_EQ(snapshot_manager->Branch(), "main"); - - // The schema manager the table keeps is the one `Table::Create` read the schema - // through, so reading the latest schema again goes through the warmed cache. - std::shared_ptr schema_manager = table->GetSchemaManager(); - ASSERT_TRUE(schema_manager != nullptr); - ASSERT_OK_AND_ASSIGN(std::optional> latest_schema, - schema_manager->Latest()); - ASSERT_TRUE(latest_schema.has_value()); - EXPECT_EQ(latest_schema.value(), table->LatestSchema()); - - // A table of a branch keeps managers bound to that branch. - ASSERT_OK_AND_ASSIGN( - std::shared_ptr
branch_table, - CreateManagedTable(fs, dir->Str() + "/branch_tbl", Identifier("db", "tbl", "dev"), - /*branch=*/"dev")); - ASSERT_TRUE(branch_table->GetSchemaManager() != nullptr); - ASSERT_TRUE(branch_table->GetSnapshotManager() != nullptr); - EXPECT_EQ(branch_table->GetSnapshotManager()->Branch(), "dev"); -} - -TEST(TableTest, TestMainBranchAliasBindsTheManagersToTheMainBranch) { +TEST(TableTest, TestMainBranchAliasResolvesToTheMainBranch) { auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); auto fs = dir->GetFileSystem(); @@ -185,11 +146,6 @@ TEST(TableTest, TestMainBranchAliasBindsTheManagersToTheMainBranch) { ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr schema, schema_manager.CreateTable( arrow::schema({arrow::field("c0", arrow::int32())}), {}, {}, {})); - // A snapshot published on the main branch, which a table opened through the alias - // reads back through the manager it keeps. - SnapshotManager snapshot_manager(fs, dir->Str()); - ASSERT_OK_AND_ASSIGN(std::string snapshot_json, BuildTestSnapshot(1).ToJsonString()); - ASSERT_OK(fs->WriteFile(snapshot_manager.SnapshotPath(1), snapshot_json, /*overwrite=*/false)); for (const auto& main_identifier : {Identifier("db", "tbl"), Identifier("db", "tbl$branch_main"), @@ -199,36 +155,15 @@ TEST(TableTest, TestMainBranchAliasBindsTheManagersToTheMainBranch) { Table::Create(fs, dir->Str(), main_identifier)); ASSERT_OK_AND_ASSIGN(std::string branch, table->CurrentBranch()); EXPECT_EQ("main", branch); - // The managers are bound to the main branch the alias spells, whose snapshots and - // schemas live under the table root itself, so both are read back. - ASSERT_TRUE(table->GetSnapshotManager() != nullptr); - EXPECT_EQ("main", table->GetSnapshotManager()->Branch()); - ASSERT_OK_AND_ASSIGN(std::optional latest_id, - table->GetSnapshotManager()->LatestSnapshotId()); - ASSERT_TRUE(latest_id.has_value()); - EXPECT_EQ(1, latest_id.value()); - ASSERT_TRUE(table->GetSchemaManager() != nullptr); - ASSERT_OK_AND_ASSIGN(std::optional> latest_schema, - table->GetSchemaManager()->Latest()); - ASSERT_TRUE(latest_schema.has_value()); - EXPECT_EQ(latest_schema.value(), table->LatestSchema()); } - // A catalog that loaded the schema itself calls the constructor with no warmed schema - // manager, which builds its managers on its own; a rest catalog is shaped like this. - // The alias binds those to the main branch as well. + // A catalog that loaded the schema itself calls the constructor directly, which a + // rest catalog is shaped like. The alias resolves to the main branch as well. ASSERT_OK_AND_ASSIGN(std::shared_ptr
plain, Table::Create(fs, dir->Str(), Identifier("db", "tbl"))); Table alias_table(fs, dir->Str(), plain->LatestSchema(), Identifier("db", "tbl$branch_MAIN")); ASSERT_OK_AND_ASSIGN(std::string alias_branch, alias_table.CurrentBranch()); EXPECT_EQ("main", alias_branch); - ASSERT_OK_AND_ASSIGN(std::optional alias_latest_id, - alias_table.GetSnapshotManager()->LatestSnapshotId()); - ASSERT_TRUE(alias_latest_id.has_value()); - EXPECT_EQ(1, alias_latest_id.value()); - ASSERT_OK_AND_ASSIGN(std::optional> alias_latest_schema, - alias_table.GetSchemaManager()->Latest()); - ASSERT_TRUE(alias_latest_schema.has_value()); } TEST(TableTest, TestCurrentBranch) { @@ -264,10 +199,6 @@ TEST(TableTest, TestCatalogOpenedTableServesThePerTableFileSystem) { // that does opens it with the file system issued for that table. ASSERT_OK_AND_ASSIGN(std::shared_ptr
opened, catalog->GetTable(identifier)); EXPECT_EQ(opened->GetFileSystem(), nullptr); - // Without a file system the table keeps no managers either, which an operator has to - // build its own in place of. - EXPECT_EQ(opened->GetSchemaManager(), nullptr); - EXPECT_EQ(opened->GetSnapshotManager(), nullptr); catalog->SetTableFileSystem(dir->GetFileSystem()); ASSERT_OK_AND_ASSIGN(opened, catalog->GetTable(identifier)); diff --git a/src/paimon/rest/rest_catalog_test.cpp b/src/paimon/rest/rest_catalog_test.cpp index 968da94a2..4b878250d 100644 --- a/src/paimon/rest/rest_catalog_test.cpp +++ b/src/paimon/rest/rest_catalog_test.cpp @@ -1065,11 +1065,10 @@ TEST_F(RestCatalogTest, BranchTableLoadsBranchSchemaFromServer) { ASSERT_NE(nullptr, main_case_schema); ASSERT_EQ((std::vector{"f0", "f1"}), main_case_schema->FieldNames()); ASSERT_EQ(0, main_case_schema->Options().count(Options::BRANCH)); - // the managers the table keeps are bound to the main branch the alias spells, not to - // a branch directory named after the alias + // the branch the alias spells resolves to the main branch, not to a branch + // directory named after the alias ASSERT_OK_AND_ASSIGN(std::string main_case_branch, main_case_table->CurrentBranch()); ASSERT_EQ("main", main_case_branch); - ASSERT_EQ("main", main_case_table->GetSnapshotManager()->Branch()); // a system table on a branch resolves against the branch's data table: the system // suffix is stripped while the branch stays in the identifier sent to the server From 6344ab6e2fd2d2f050294e30e3d18ca52644a456 Mon Sep 17 00:00:00 2001 From: "yonghao.fyh" Date: Mon, 19 Oct 2026 15:08:56 +0800 Subject: [PATCH 3/6] fix --- .../core/operation/file_store_commit.cpp | 346 ----- .../core/operation/file_store_write.cpp | 463 ------ src/paimon/core/table/source/table_read.cpp | 211 --- src/paimon/core/table/source/table_scan.cpp | 436 ------ src/paimon/core/table/table.cpp | 1344 +++++++++++++++++ 5 files changed, 1344 insertions(+), 1456 deletions(-) diff --git a/src/paimon/core/operation/file_store_commit.cpp b/src/paimon/core/operation/file_store_commit.cpp index 457b48dd7..4e0747437 100644 --- a/src/paimon/core/operation/file_store_commit.cpp +++ b/src/paimon/core/operation/file_store_commit.cpp @@ -18,349 +18,3 @@ */ #include "paimon/file_store_commit.h" - -#include -#include -#include -#include -#include - -#include "fmt/format.h" -#include "paimon/catalog/catalog.h" -#include "paimon/commit_context.h" -#include "paimon/common/types/data_field.h" -#include "paimon/common/utils/binary_row_partition_computer.h" -#include "paimon/core/catalog/catalog_snapshot_commit.h" -#include "paimon/core/catalog/catalog_utils.h" -#include "paimon/core/catalog/renaming_snapshot_commit.h" -#include "paimon/core/catalog/snapshot_commit.h" -#include "paimon/core/catalog/version_managed_catalog.h" -#include "paimon/core/core_options.h" -#include "paimon/core/manifest/index_manifest_entry.h" -#include "paimon/core/manifest/index_manifest_file.h" -#include "paimon/core/manifest/manifest_file.h" -#include "paimon/core/manifest/manifest_list.h" -#include "paimon/core/operation/append_only_file_store_scan.h" -#include "paimon/core/operation/expire_snapshots.h" -#include "paimon/core/operation/file_store_commit_impl.h" -#include "paimon/core/operation/file_store_scan.h" -#include "paimon/core/operation/key_value_file_store_scan.h" -#include "paimon/core/schema/schema_manager.h" -#include "paimon/core/schema/schema_validation.h" -#include "paimon/core/schema/table_schema.h" -#include "paimon/core/table/format/format_table_file_store_commit.h" -#include "paimon/core/table/format/format_table_loader.h" -#include "paimon/core/utils/branch_manager.h" -#include "paimon/core/utils/field_mapping.h" -#include "paimon/core/utils/file_store_path_factory.h" -#include "paimon/core/utils/snapshot_manager.h" -#include "paimon/defs.h" -#include "paimon/format/file_format.h" -#include "paimon/fs/file_system.h" -#include "paimon/result.h" -#include "paimon/schema/schema.h" -#include "paimon/table/format/format_table.h" - -namespace arrow { -class Schema; -} // namespace arrow - -namespace paimon { - -namespace { - -CommitScanner::ScanSupplier CreateAppendScanSupplier( - const std::shared_ptr& snapshot_manager, - const std::shared_ptr& schema_manager, - const std::shared_ptr& manifest_list, - const std::shared_ptr& manifest_file, - const std::shared_ptr& table_schema, - const std::shared_ptr& arrow_schema, const CoreOptions& options, - const std::shared_ptr& executor, const std::shared_ptr& pool) { - return [snapshot_manager, schema_manager, manifest_list, manifest_file, table_schema, - arrow_schema, options, executor, pool](const std::shared_ptr& scan_filter) - -> Result> { - PAIMON_ASSIGN_OR_RAISE( - std::unique_ptr scan, - AppendOnlyFileStoreScan::Create(snapshot_manager, schema_manager, manifest_list, - manifest_file, table_schema, arrow_schema, scan_filter, - options, executor, pool)); - return std::unique_ptr(std::move(scan)); - }; -} - -CommitScanner::ScanSupplier CreatePkScanSupplier( - const std::shared_ptr& snapshot_manager, - const std::shared_ptr& schema_manager, - const std::shared_ptr& manifest_list, - const std::shared_ptr& manifest_file, - const std::shared_ptr& table_schema, - const std::shared_ptr& arrow_schema, const CoreOptions& options, - const std::shared_ptr& executor, const std::shared_ptr& pool) { - return [snapshot_manager, schema_manager, manifest_list, manifest_file, table_schema, - arrow_schema, options, executor, pool](const std::shared_ptr& scan_filter) - -> Result> { - PAIMON_ASSIGN_OR_RAISE( - std::unique_ptr scan, - KeyValueFileStoreScan::Create(snapshot_manager, schema_manager, manifest_list, - manifest_file, table_schema, arrow_schema, scan_filter, - options, executor, pool)); - return std::unique_ptr(std::move(scan)); - }; -} - -bool IsCatalogCommit(const CommitContext& ctx) { - return AsVersionManaged(ctx.GetCatalog()) != nullptr; -} - -Result> NewSnapshotCommit( - const CommitContext& ctx, const std::optional& catalog_identifier, - const std::shared_ptr& fs, - const std::shared_ptr& snapshot_manager) { - if (IsCatalogCommit(ctx)) { - if (!catalog_identifier) { - return Status::Invalid("a catalog commit requires a table identifier"); - } - // Reuse the caller's UUID so table recreation cannot redirect this commit. - return std::shared_ptr(std::make_shared( - ctx.GetCatalog(), catalog_identifier.value(), ctx.GetTableId())); - } - if (ctx.UseRESTCatalogCommit()) { - return std::shared_ptr( - std::make_shared(ctx.GetTableId())); - } - // Catalogs without version management may still supply a table ID. - if (ctx.GetTableId() && ctx.GetCatalog() == nullptr) { - return Status::Invalid( - "a commit which publishes the snapshot by writing it to the table directory sends no " - "commit table request, so there is nowhere to put a table id; commit through a " - "catalog with WithCatalog() or ask for the request with UseRESTCatalogCommit()"); - } - return std::shared_ptr( - std::make_shared(fs, snapshot_manager)); -} - -/// Creates a format-table commit, rejecting non-default options that require snapshots. -Result> NewFormatTableCommit( - const std::shared_ptr& table, const CommitContext& ctx) { - if (!ctx.IgnoreEmptyCommit()) { - return Status::NotImplemented( - "a format table cannot record an empty commit: keeping one means writing a " - "snapshot that adds no files, and there are no snapshots here"); - } - if (ctx.UseRESTCatalogCommit()) { - return Status::NotImplemented( - "a format table commits by renaming files into place, not by sending a snapshot " - "to a rest catalog"); - } - if (ctx.GetTableId()) { - return Status::NotImplemented( - "a format table sends no commit table request, so there is nowhere to put a table id"); - } - if (ctx.GetCatalog() != nullptr) { - return Status::NotImplemented( - "a format table keeps no snapshot for a catalog to take, so it commits by renaming " - "files into place rather than through the catalog it was loaded from"); - } - if (ctx.AppendCommitCheckConflict()) { - return Status::NotImplemented( - "a format table has no manifests to check a concurrent commit against"); - } - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr format_commit, - FormatTableFileStoreCommit::Create(table)); - return std::unique_ptr(std::move(format_commit)); -} - -} // namespace - -Result> FileStoreCommit::Create( - std::unique_ptr ctx) { - if (ctx == nullptr) { - return Status::Invalid("commit context is null pointer"); - } - if (ctx->GetMemoryPool() == nullptr) { - return Status::Invalid("memory pool is null pointer"); - } - if (ctx->GetExecutor() == nullptr) { - return Status::Invalid("executor is null pointer"); - } - - // A table the caller already loaded says what it is, so nothing is read to find out. - if (ctx->GetFormatTable() != nullptr) { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr given_table, - FormatTable::Copy(ctx->GetFormatTable(), ctx->GetOptions())); - return NewFormatTableCommit(given_table, *ctx); - } - - PAIMON_RETURN_NOT_OK(CheckVersionManagementImplemented(ctx->GetCatalog())); - // Use catalog credentials unless the caller supplied a file system. - std::shared_ptr specific_fs = ctx->GetSpecificFileSystem(); - if (specific_fs == nullptr && ctx->GetCatalog() != nullptr) { - if (!ctx->GetIdentifier()) { - return Status::Invalid("a catalog commit requires a table identifier"); - } - // Use the credentials of this table, which a catalog issuing per-table temporary - // ones only hands out through GetTableFileSystem. - PAIMON_ASSIGN_OR_RAISE(specific_fs, - ctx->GetCatalog()->GetTableFileSystem(ctx->GetIdentifier().value())); - } - PAIMON_ASSIGN_OR_RAISE(CoreOptions tmp_options, - CoreOptions::FromMap(ctx->GetOptions(), specific_fs)); - const std::string& root_path = ctx->GetRootPath(); - // The branch the context was built for, already validated by `CommitContextBuilder`. - PAIMON_ASSIGN_OR_RAISE(std::string branch, BranchManager::ResolveBranch( - ctx->GetIdentifier(), ctx->GetOptions(), - /*explicit_branch=*/std::nullopt, "commit")); - std::optional catalog_identifier; - if (ctx->GetIdentifier()) { - PAIMON_ASSIGN_OR_RAISE( - Identifier branch_identifier, - CatalogUtils::BranchIdentifier(ctx->GetIdentifier().value(), branch)); - catalog_identifier.emplace(std::move(branch_identifier)); - } - // Catalog-managed schemas may be absent from the table directory, but only on the main branch: - // a branch publishes its own under `branch/branch-`, which is where the load below and a - // read of the branch both read it, so a catalog answering for no branch schema does not fail - // a commit to one. - std::optional catalog_table_schema; - FileStoreCommitImpl::SchemaIdLoader schema_id_loader; - if (ctx->GetCatalog() != nullptr) { - if (!catalog_identifier) { - return Status::Invalid("a catalog commit requires a table identifier"); - } - if (BranchManager::IsMainBranch(branch)) { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr loaded_schema, - ctx->GetCatalog()->LoadTableSchema(catalog_identifier.value())); - PAIMON_ASSIGN_OR_RAISE(std::string schema_json, loaded_schema->GetJsonSchema()); - catalog_table_schema = std::move(schema_json); - } - std::shared_ptr catalog = ctx->GetCatalog(); - Identifier identifier = catalog_identifier.value(); - schema_id_loader = [catalog, identifier]() -> Result { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, - catalog->LoadTableSchema(identifier)); - std::shared_ptr data_schema = std::dynamic_pointer_cast(schema); - if (data_schema == nullptr) { - return Status::Invalid( - "the catalog did not hand back a data table schema, which has the id a " - "snapshot records"); - } - return data_schema->Id(); - }; - } - // A format table commits by renaming files into place, so it never reaches the snapshot path - // below. The managed path here reads the commit's branch, so this reads the same one: the two - // must not dispatch on different schemas. - auto schema_manager = - std::make_shared(tmp_options.GetFileSystem(), root_path, branch); - std::shared_ptr latest_schema; - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr format_table, - FormatTableLoader::TryLoad( - tmp_options.GetFileSystem(), root_path, branch, ctx->GetOptions(), - catalog_table_schema, schema_manager.get(), &latest_schema)); - if (format_table != nullptr) { - return NewFormatTableCommit(format_table, *ctx); - } - if (latest_schema == nullptr) { - return Status::Invalid(fmt::format("not found latest schema in branch {}", branch)); - } - const std::shared_ptr& schema = latest_schema; - auto opts = schema->Options(); - for (const auto& [key, value] : ctx->GetOptions()) { - opts[key] = value; - } - // The metadata paths below are built for this branch, so what reads through these options, - // whose cache keys are scoped by it, has to read the same one. - opts[Options::BRANCH] = branch; - std::shared_ptr arrow_schema = - DataField::ConvertDataFieldsToArrowSchema(schema->Fields()); - PAIMON_ASSIGN_OR_RAISE(CoreOptions options, CoreOptions::FromMap(opts, specific_fs)); - PAIMON_RETURN_NOT_OK(SchemaValidation::ValidateRealtime(*schema, options)); - assert(options.GetFileSystem()); - assert(options.GetFileFormat()); - PAIMON_RETURN_NOT_OK(FileStoreCommitImpl::ValidateCommitOptions(options)); - PAIMON_ASSIGN_OR_RAISE(bool is_object_store, FileSystem::IsObjectStore(root_path)); - // Catalog commits do not require atomic rename on the object store. - if (is_object_store && !ctx->UseRESTCatalogCommit() && !IsCatalogCommit(*ctx) && - opts.find("enable-object-store-commit-in-inte-test") == opts.end()) { - return Status::NotImplemented( - "commit operation does not support object store file system for now"); - } - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr partition_computer, - BinaryRowPartitionComputer::Create(schema->PartitionKeys(), arrow_schema, - options.GetPartitionDefaultName(), - options.LegacyPartitionNameEnabled(), - ctx->GetMemoryPool())); - PAIMON_ASSIGN_OR_RAISE(std::vector external_paths, options.CreateExternalPaths()); - PAIMON_ASSIGN_OR_RAISE(std::optional global_index_external_path, - options.CreateGlobalIndexExternalPath()); - - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr path_factory, - FileStorePathFactory::Create( - root_path, arrow_schema, schema->PartitionKeys(), options.GetPartitionDefaultName(), - options.GetFileFormat()->Identifier(), options.DataFilePrefix(), - options.LegacyPartitionNameEnabled(), external_paths, global_index_external_path, - options.IndexFileInDataFileDir(), ctx->GetMemoryPool())); - - auto snapshot_manager = - std::make_shared(options.GetFileSystem(), root_path, branch); - if (IsCatalogCommit(*ctx)) { - std::shared_ptr catalog = ctx->GetCatalog(); - VersionManagedCatalog* versioned = AsVersionManaged(catalog); - Identifier identifier = catalog_identifier.value(); - // Capture the catalog to keep the versioned interface alive. - snapshot_manager->SetSnapshotLoader( - [catalog, versioned, identifier]() -> Result> { - return versioned->LoadSnapshot(identifier); - }); - } - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr manifest_list, - ManifestList::Create(options.GetFileSystem(), options.GetManifestFormat(), - options.GetManifestCompression(), path_factory, options.GetCache(), - ctx->GetMemoryPool())); - - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr partition_schema, - FieldMapping::GetPartitionSchema(arrow_schema, schema->PartitionKeys())); - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr manifest_file, - ManifestFile::Create(options.GetFileSystem(), options.GetManifestFormat(), - options.GetManifestCompression(), path_factory, - options.GetManifestTargetFileSize(), ctx->GetMemoryPool(), options, - partition_schema)); - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr index_manifest_file, - IndexManifestFile::Create(options.GetFileSystem(), options.GetManifestFormat(), - options.GetManifestCompression(), path_factory, - options.GetBucket(), ctx->GetMemoryPool(), options)); - - auto expire_snapshots = std::make_shared( - snapshot_manager, path_factory, manifest_list, manifest_file, index_manifest_file, - options.GetFileSystem(), options.GetExpireConfig(), options.RealtimeEnabled(), - ctx->GetExecutor()); - - CommitScanner::ScanSupplier scan_supplier; - if (schema->PrimaryKeys().empty()) { - scan_supplier = CreateAppendScanSupplier(snapshot_manager, schema_manager, manifest_list, - manifest_file, schema, arrow_schema, options, - ctx->GetExecutor(), ctx->GetMemoryPool()); - } else { - scan_supplier = CreatePkScanSupplier(snapshot_manager, schema_manager, manifest_list, - manifest_file, schema, arrow_schema, options, - ctx->GetExecutor(), ctx->GetMemoryPool()); - } - - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr snapshot_commit, - NewSnapshotCommit(*ctx, catalog_identifier, options.GetFileSystem(), snapshot_manager)); - - return std::make_unique( - ctx->GetMemoryPool(), ctx->GetExecutor(), arrow_schema, root_path, ctx->GetCommitUser(), - options, path_factory, std::move(partition_computer), snapshot_manager, snapshot_commit, - ctx->IgnoreEmptyCommit(), ctx->AppendCommitCheckConflict(), schema, - std::move(schema_id_loader), manifest_file, manifest_list, index_manifest_file, - expire_snapshots, schema_manager, std::move(scan_supplier)); -} - -} // namespace paimon diff --git a/src/paimon/core/operation/file_store_write.cpp b/src/paimon/core/operation/file_store_write.cpp index 244a5c485..c1b889473 100644 --- a/src/paimon/core/operation/file_store_write.cpp +++ b/src/paimon/core/operation/file_store_write.cpp @@ -18,139 +18,7 @@ #include "paimon/file_store_write.h" -#include -#include -#include -#include -#include - -#include "fmt/format.h" -#include "paimon/catalog/catalog.h" -#include "paimon/common/global_index/btree/btree_defs.h" -#include "paimon/common/types/data_field.h" -#include "paimon/common/utils/fields_comparator.h" -#include "paimon/common/utils/string_utils.h" -#include "paimon/core/catalog/catalog_utils.h" -#include "paimon/core/catalog/version_managed_catalog.h" -#include "paimon/core/core_options.h" -#include "paimon/core/disk/io_manager.h" -#include "paimon/core/index/index_file_handler.h" -#include "paimon/core/index/pk/bucketed_primary_key_index_maintainer.h" -#include "paimon/core/index/pk/primary_key_index_definitions.h" -#include "paimon/core/manifest/index_manifest_file.h" -#include "paimon/core/mergetree/compact/lookup_merge_function.h" -#include "paimon/core/mergetree/compact/merge_function.h" -#include "paimon/core/mergetree/compact/reducer_merge_function_wrapper.h" -#include "paimon/core/operation/append_only_file_store_write.h" -#include "paimon/core/operation/commit/realtime_commit_properties.h" -#include "paimon/core/operation/key_value_file_store_write.h" -#include "paimon/core/options/merge_engine.h" -#include "paimon/core/postpone/postpone_bucket_file_store_write.h" -#include "paimon/core/realtime/realtime_context_impl.h" -#include "paimon/core/realtime/realtime_schema_layout.h" -#include "paimon/core/schema/schema_manager.h" -#include "paimon/core/schema/schema_validation.h" -#include "paimon/core/schema/table_schema.h" -#include "paimon/core/table/bucket_mode.h" -#include "paimon/core/table/format/format_table_file_store_write.h" -#include "paimon/core/table/format/format_table_loader.h" -#include "paimon/core/utils/branch_manager.h" -#include "paimon/core/utils/field_mapping.h" -#include "paimon/core/utils/file_store_path_factory.h" -#include "paimon/core/utils/primary_key_table_utils.h" -#include "paimon/core/utils/snapshot_manager.h" -#include "paimon/defs.h" -#include "paimon/format/file_format.h" -#include "paimon/result.h" -#include "paimon/schema/schema.h" -#include "paimon/table/format/format_table.h" -#include "paimon/write_context.h" - -namespace arrow { -class Schema; -} // namespace arrow - namespace paimon { -struct KeyValue; -template -class MergeFunctionWrapper; - -namespace { - -Status RestoreRealtimeCommittedProgress(const std::shared_ptr& realtime_context, - const std::shared_ptr& snapshot_manager, - const CoreOptions& options) { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr realtime_context_impl, - RealtimeContextImpl::Cast(realtime_context)); - PAIMON_RETURN_NOT_OK(realtime_context_impl->CheckUsable()); - PAIMON_ASSIGN_OR_RAISE(std::optional latest_snapshot, - snapshot_manager->LatestSnapshot()); - if (latest_snapshot) { - PAIMON_ASSIGN_OR_RAISE( - RealtimeOffsetMap realtime_committed_offsets, - RealtimeCommitProperties::ReadOffsets(latest_snapshot, options.GetFileSystem())); - PAIMON_RETURN_NOT_OK(realtime_context_impl->AdvanceCommittedProgress( - latest_snapshot->Id(), realtime_committed_offsets, /*committed_data_files=*/{})); - } - return Status::OK(); -} - -/// Maps a `WriteContext` onto `FormatTableFileStoreWrite`, refusing by name what a format table -/// cannot honour rather than silently dropping it. -Result> NewFormatTableWrite( - const std::shared_ptr& table, const WriteContext& ctx) { - if (ctx.GetCatalog() != nullptr) { - return Status::Invalid( - "WithCatalog() requires a native table; use WriteContextBuilder(FormatTable) for a " - "format table"); - } - if (ctx.IsStreamingMode()) { - return Status::NotImplemented( - "a format table has no snapshots, so there is nothing a streaming write could " - "commit against"); - } - if (ctx.GetRealtimeContext() != nullptr) { - return Status::NotImplemented("a format table has no real-time store to write into"); - } - if (!ctx.GetWriteSchema().empty()) { - return Status::NotImplemented( - "a format table write takes the table's own columns; a write schema naming a " - "subset of them is not supported yet"); - } - // A write id prefixes a postpone-bucket writer's files so that one compaction reader can - // put them back in order. A format table has no buckets, so it would identify nothing. - if (ctx.GetWriteId().has_value()) { - return Status::NotImplemented( - "a format table has no buckets, so a write id would name nothing"); - } - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr format_write, - FormatTableFileStoreWrite::Create(table, ctx.GetMemoryPool())); - return std::unique_ptr(std::move(format_write)); -} - -Result HasHistoricalPrimaryKeyBTreeDefinition( - const std::shared_ptr& schema_manager, int64_t current_schema_id) { - if (current_schema_id <= TableSchema::FIRST_SCHEMA_ID) { - return false; - } - PAIMON_ASSIGN_OR_RAISE(std::vector schema_ids, schema_manager->ListAllIds()); - for (int64_t schema_id : schema_ids) { - if (schema_id >= current_schema_id) { - continue; - } - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr historical_schema, - schema_manager->ReadSchema(schema_id)); - const auto& historical_options = historical_schema->Options(); - auto option = historical_options.find(Options::PK_BTREE_INDEX_COLUMNS); - if (option != historical_options.end() && - !StringUtils::IsNullOrWhitespaceOnly(option->second)) { - return true; - } - } - return false; -} - -} // namespace Result> FileStoreWrite::PrepareCommitWithProgress(int64_t) { return Status::Invalid("prepare commit with progress requires a real-time writer"); @@ -164,335 +32,4 @@ Status FileStoreWrite::RefreshCommittedSnapshot(int64_t) { return Status::Invalid("refresh committed snapshot requires a real-time writer"); } -Result> FileStoreWrite::Create(std::unique_ptr ctx) { - if (ctx == nullptr) { - return Status::Invalid("write context is null pointer"); - } - if (ctx->GetMemoryPool() == nullptr) { - return Status::Invalid("memory pool is null pointer"); - } - if (ctx->GetExecutor() == nullptr) { - return Status::Invalid("executor is null pointer"); - } - - // A table the caller already loaded says what it is, so nothing is read to find out. - if (ctx->GetFormatTable() != nullptr) { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr given_table, - FormatTable::Copy(ctx->GetFormatTable(), ctx->GetOptions())); - return NewFormatTableWrite(given_table, *ctx); - } - - PAIMON_RETURN_NOT_OK(CheckVersionManagementImplemented(ctx->GetCatalog())); - std::shared_ptr specific_fs = ctx->GetSpecificFileSystem(); - if (specific_fs == nullptr && ctx->GetFileSystemSchemeToIdentifierMap().empty() && - ctx->GetCatalog() != nullptr) { - if (!ctx->GetIdentifier()) { - return Status::Invalid("a catalog write requires a table identifier"); - } - // A catalog issuing per-table temporary credentials only hands them out through - // GetTableFileSystem, so the write uses the credentials of this table. - PAIMON_ASSIGN_OR_RAISE(specific_fs, - ctx->GetCatalog()->GetTableFileSystem(ctx->GetIdentifier().value())); - } - PAIMON_ASSIGN_OR_RAISE(CoreOptions tmp_options, - CoreOptions::FromMap(ctx->GetOptions(), specific_fs, - ctx->GetFileSystemSchemeToIdentifierMap())); - std::string branch = ctx->GetBranch(); - std::optional catalog_identifier; - if (ctx->GetIdentifier()) { - PAIMON_ASSIGN_OR_RAISE( - Identifier branch_identifier, - CatalogUtils::BranchIdentifier(ctx->GetIdentifier().value(), branch)); - catalog_identifier.emplace(std::move(branch_identifier)); - } - // Only the main branch may have a current schema the catalog alone holds. A branch publishes - // its own under `branch/branch-`, which is where the load below and a read of the branch - // both read it, so a catalog answering for no branch schema does not fail a write to one. - std::optional catalog_table_schema; - if (ctx->GetCatalog() != nullptr) { - if (!catalog_identifier) { - return Status::Invalid("a catalog write requires a table identifier"); - } - if (BranchManager::IsMainBranch(branch)) { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, - ctx->GetCatalog()->LoadTableSchema(catalog_identifier.value())); - PAIMON_ASSIGN_OR_RAISE(catalog_table_schema, schema->GetJsonSchema()); - } - } - // A format table writes plain data files into a directory, so it never reaches the manifest - // path below. - auto schema_manager = - std::make_shared(tmp_options.GetFileSystem(), ctx->GetRootPath(), branch); - std::shared_ptr latest_schema; - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr format_table, - FormatTableLoader::TryLoad(tmp_options.GetFileSystem(), ctx->GetRootPath(), branch, - ctx->GetOptions(), catalog_table_schema, schema_manager.get(), - &latest_schema)); - if (format_table != nullptr) { - return NewFormatTableWrite(format_table, *ctx); - } - // The schema the dispatch above already read through `schema_manager`, rather than a second - // read of the same file. - if (latest_schema == nullptr) { - return Status::Invalid(fmt::format("not found latest schema in branch {}", branch)); - } - const std::shared_ptr& schema = latest_schema; - auto arrow_schema = DataField::ConvertDataFieldsToArrowSchema(schema->Fields()); - auto opts = schema->Options(); - for (const auto& [key, value] : ctx->GetOptions()) { - opts[key] = value; - } - // The metadata paths above are built for this branch, so what reads through these options, - // whose cache keys are scoped by it, has to read the same one. - opts[Options::BRANCH] = branch; - PAIMON_ASSIGN_OR_RAISE( - CoreOptions options, - CoreOptions::FromMap(opts, specific_fs, ctx->GetFileSystemSchemeToIdentifierMap())); - PAIMON_RETURN_NOT_OK(SchemaValidation::ValidateRealtime(*schema, options)); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr partition_schema, - FieldMapping::GetPartitionSchema(arrow_schema, schema->PartitionKeys())); - - PAIMON_ASSIGN_OR_RAISE(std::vector external_paths, options.CreateExternalPaths()); - PAIMON_ASSIGN_OR_RAISE(std::optional global_index_external_path, - options.CreateGlobalIndexExternalPath()); - - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr file_store_path_factory, - FileStorePathFactory::Create(ctx->GetRootPath(), arrow_schema, schema->PartitionKeys(), - options.GetPartitionDefaultName(), - options.GetWriteFileFormat(/*level=*/0)->Identifier(), - options.DataFilePrefix(), options.LegacyPartitionNameEnabled(), - external_paths, global_index_external_path, - options.IndexFileInDataFileDir(), ctx->GetMemoryPool())); - auto snapshot_manager = - std::make_shared(options.GetFileSystem(), ctx->GetRootPath(), branch); - if (VersionManagedCatalog* versioned = AsVersionManaged(ctx->GetCatalog())) { - std::shared_ptr catalog = ctx->GetCatalog(); - Identifier identifier = catalog_identifier.value(); - // Keep the catalog alive while the writer uses its snapshot loader. - snapshot_manager->SetSnapshotLoader( - [catalog, versioned, identifier]() -> Result> { - return versioned->LoadSnapshot(identifier); - }); - } - std::shared_ptr io_manager; - const auto& io_temp_dir = ctx->GetTempDirectory(); - if (!io_temp_dir.empty()) { - io_manager = std::make_shared(io_temp_dir, options.GetFileSystem()); - } - - bool ignore_previous_files = ctx->IgnorePreviousFiles(); - if (ctx->GetRealtimeContext() && !options.RealtimeEnabled()) { - return Status::Invalid("real-time write requires realtime.enabled=true"); - } - if (ctx->GetRealtimeContext()) { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr realtime_context_impl, - RealtimeContextImpl::Cast(ctx->GetRealtimeContext())); - PAIMON_RETURN_NOT_OK(realtime_context_impl->CheckUsable()); - } - if (schema->PrimaryKeys().empty()) { - // append table - const bool realtime_deduplicate = !options.GetRealtimeDeduplicateKeyFields().empty(); - bool need_dv_maintainer_factory = options.DeletionVectorsEnabled(); - if (options.GetBucket() == -1) { - need_dv_maintainer_factory = false; - ignore_previous_files = true; - } else if (options.GetBucket() <= 0) { - return Status::Invalid( - fmt::format("not support bucket {} in append table", options.GetBucket())); - } - if (ctx->GetRealtimeContext()) { - if (!ctx->GetWriteSchema().empty()) { - return Status::Invalid("real-time append write does not support a custom schema"); - } - if (!realtime_deduplicate) { - PAIMON_RETURN_NOT_OK(RestoreRealtimeCommittedProgress(ctx->GetRealtimeContext(), - snapshot_manager, options)); - } - } else if (realtime_deduplicate) { - return Status::Invalid("real-time deduplicate write requires a real-time context"); - } - std::shared_ptr write_schema = arrow_schema; - const auto& write_field_names = ctx->GetWriteSchema(); - if (!write_field_names.empty()) { - arrow::FieldVector write_fields; - write_fields.reserve(write_field_names.size()); - for (const auto& field_name : write_field_names) { - auto field = arrow_schema->GetFieldByName(field_name); - if (!field) { - // TODO(xinyu.lxy): support _ROW_ID and _SEQUENCE_NUMBER - return Status::Invalid( - fmt::format("write field {} does not exist in table schema", field_name)); - } - write_fields.push_back(field); - } - write_schema = arrow::schema(write_fields); - } - - std::shared_ptr realtime_schema_layout; - if (ctx->GetRealtimeContext()) { - const RealtimeStoreMode mode = realtime_deduplicate ? RealtimeStoreMode::DEDUPLICATE - : RealtimeStoreMode::APPEND_ONLY; - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr layout, - RealtimeSchemaLayout::Create(mode, write_schema)); - realtime_schema_layout = std::move(layout); - } - - std::shared_ptr dv_maintainer_factory; - if (need_dv_maintainer_factory) { - PAIMON_ASSIGN_OR_RAISE( - std::unique_ptr index_manifest_file, - IndexManifestFile::Create(options.GetFileSystem(), options.GetManifestFormat(), - options.GetManifestCompression(), file_store_path_factory, - options.GetBucket(), ctx->GetMemoryPool(), options)); - auto index_file_handler = std::make_shared( - options.GetFileSystem(), std::move(index_manifest_file), - std::make_shared(file_store_path_factory), - options.DeletionVectorsBitmap64(), ctx->GetMemoryPool()); - dv_maintainer_factory = - std::make_shared(index_file_handler); - } - - auto file_store_write = std::make_unique( - file_store_path_factory, snapshot_manager, schema_manager, ctx->GetCommitUser(), - ctx->GetRootPath(), schema, arrow_schema, write_schema, realtime_schema_layout, - partition_schema, dv_maintainer_factory, io_manager, options, ignore_previous_files, - ctx->IsStreamingMode(), ctx->IgnoreNumBucketCheck(), ctx->GetRealtimeContext(), - ctx->GetExecutor(), ctx->GetMemoryPool()); - if (ctx->GetRealtimeContext() && realtime_deduplicate) { - PAIMON_ASSIGN_OR_RAISE(std::optional latest_snapshot, - snapshot_manager->LatestSnapshot()); - if (latest_snapshot) { - PAIMON_RETURN_NOT_OK( - file_store_write->RefreshCommittedSnapshot(latest_snapshot->Id())); - } - } - return std::unique_ptr(std::move(file_store_write)); - } else { - // pk table - if (ctx->GetRealtimeContext()) { - PAIMON_RETURN_NOT_OK(PrimaryKeyTableUtils::ValidateRealtimeOptions(options, *schema)); - if (ignore_previous_files) { - return Status::NotImplemented( - "PK realtime requires restore from the latest snapshot"); - } - if (!ctx->GetWriteSchema().empty()) { - return Status::NotImplemented("PK realtime does not support a custom write schema"); - } - PAIMON_RETURN_NOT_OK(RestoreRealtimeCommittedProgress(ctx->GetRealtimeContext(), - snapshot_manager, options)); - } - if (options.GetBucket() == BucketModeDefine::POSTPONE_BUCKET) { - return PostponeBucketFileStoreWrite::Create( - snapshot_manager, schema_manager, ctx->GetCommitUser(), ctx->GetRootPath(), schema, - arrow_schema, partition_schema, io_manager, options, ctx->IsStreamingMode(), - ctx->IgnoreNumBucketCheck(), ctx->GetWriteId(), - ctx->GetFileSystemSchemeToIdentifierMap(), ctx->GetExecutor(), ctx->GetMemoryPool(), - specific_fs); - } - if (options.GetBucket() <= 0) { - return Status::Invalid( - fmt::format("not support bucket {} in key value table", options.GetBucket())); - } - PAIMON_ASSIGN_OR_RAISE(std::vector trimmed_primary_keys, - schema->TrimmedPrimaryKeys()); - PAIMON_ASSIGN_OR_RAISE(std::vector trimmed_primary_key_fields, - schema->GetFields(trimmed_primary_keys)); - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr key_comparator, - FieldsComparator::Create(trimmed_primary_key_fields, - options.SequenceFieldSortOrderIsAscending())); - auto primary_keys = schema->PrimaryKeys(); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr merge_function, - PrimaryKeyTableUtils::CreateMergeFunction( - arrow_schema, primary_keys, options, ctx->GetMemoryPool())); - if (options.NeedLookup() && options.GetMergeEngine() != MergeEngine::FIRST_ROW) { - // don't wrap first row, it is already OK - merge_function = std::make_unique(std::move(merge_function)); - } - std::shared_ptr> merge_function_wrapper = - std::make_shared(std::move(merge_function)); - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr sequence_fields_comparator, - PrimaryKeyTableUtils::CreateSequenceFieldsComparator(schema->Fields(), options)); - - std::shared_ptr dv_maintainer_factory; - PAIMON_ASSIGN_OR_RAISE(PrimaryKeyIndexDefinitions primary_key_index_definitions, - PrimaryKeyIndexDefinitions::Create(*schema)); - bool has_btree_index = false; - for (const PrimaryKeyIndexDefinition& definition : - primary_key_index_definitions.Definitions()) { - if (definition.GetFamily() == PrimaryKeyIndexDefinition::Family::BTREE) { - has_btree_index = true; - break; - } - } - std::optional latest_snapshot; - bool has_existing_index_manifest = false; - if (!has_btree_index) { - PAIMON_ASSIGN_OR_RAISE( - bool has_historical_btree_index, - HasHistoricalPrimaryKeyBTreeDefinition(schema_manager, schema->Id())); - if (has_historical_btree_index) { - PAIMON_ASSIGN_OR_RAISE(latest_snapshot, snapshot_manager->LatestSnapshot()); - has_existing_index_manifest = - latest_snapshot.has_value() && latest_snapshot->IndexManifest().has_value(); - } - } - std::shared_ptr index_file_handler; - if (options.DeletionVectorsEnabled() || has_btree_index || has_existing_index_manifest) { - PAIMON_ASSIGN_OR_RAISE( - std::unique_ptr index_manifest_file, - IndexManifestFile::Create(options.GetFileSystem(), options.GetManifestFormat(), - options.GetManifestCompression(), file_store_path_factory, - options.GetBucket(), ctx->GetMemoryPool(), options)); - index_file_handler = std::make_shared( - options.GetFileSystem(), std::move(index_manifest_file), - std::make_shared(file_store_path_factory), - options.DeletionVectorsBitmap64(), ctx->GetMemoryPool()); - } - bool has_restored_btree_index = false; - if (!has_btree_index && has_existing_index_manifest) { - PAIMON_ASSIGN_OR_RAISE( - std::vector restored_btree_entries, - index_file_handler->Scan( - latest_snapshot.value(), [](const IndexManifestEntry& entry) -> bool { - return entry.index_file->IndexType() == BtreeDefs::kIdentifier && - IndexFileHandler::IsPrimaryKeySourceIndex(*entry.index_file); - })); - has_restored_btree_index = !restored_btree_entries.empty(); - } - if (options.DeletionVectorsEnabled()) { - dv_maintainer_factory = - std::make_shared(index_file_handler); - } - std::shared_ptr - primary_key_index_maintainer_factory; - if (has_btree_index || has_restored_btree_index) { - primary_key_index_maintainer_factory = - std::make_shared( - ctx->GetRootPath(), branch, schema, primary_key_index_definitions.Definitions(), - file_store_path_factory, index_file_handler, options, io_manager, - ctx->EnableMultiThreadSpill(), ctx->GetExecutor(), ctx->GetMemoryPool()); - } - - std::shared_ptr realtime_schema_layout; - if (ctx->GetRealtimeContext()) { - PAIMON_ASSIGN_OR_RAISE( - std::unique_ptr layout, - RealtimeSchemaLayout::Create(RealtimeStoreMode::PRIMARY_KEY, arrow_schema)); - realtime_schema_layout = std::move(layout); - } - - return std::make_unique( - file_store_path_factory, snapshot_manager, schema_manager, ctx->GetCommitUser(), - ctx->GetRootPath(), schema, arrow_schema, realtime_schema_layout, partition_schema, - dv_maintainer_factory, primary_key_index_maintainer_factory, io_manager, key_comparator, - sequence_fields_comparator, merge_function_wrapper, options, ignore_previous_files, - ctx->IsStreamingMode(), ctx->IgnoreNumBucketCheck(), ctx->EnableMultiThreadSpill(), - ctx->GetRealtimeContext(), ctx->GetExecutor(), ctx->GetMemoryPool()); - } -} - } // namespace paimon diff --git a/src/paimon/core/table/source/table_read.cpp b/src/paimon/core/table/source/table_read.cpp index 7dd10e36b..e723c54bb 100644 --- a/src/paimon/core/table/source/table_read.cpp +++ b/src/paimon/core/table/source/table_read.cpp @@ -19,218 +19,7 @@ #include "paimon/table/source/table_read.h" -#include -#include -#include -#include -#include -#include -#include - -#include "arrow/c/bridge.h" -#include "fmt/format.h" -#include "paimon/common/types/data_field.h" -#include "paimon/common/utils/arrow/status_utils.h" -#include "paimon/common/utils/string_utils.h" -#include "paimon/core/core_options.h" -#include "paimon/core/operation/internal_read_context.h" -#include "paimon/core/schema/schema_manager.h" -#include "paimon/core/schema/schema_validation.h" -#include "paimon/core/schema/table_schema.h" -#include "paimon/core/table/format/format_table_loader.h" -#include "paimon/core/table/format/format_table_read.h" -#include "paimon/core/table/source/append_only_table_read.h" -#include "paimon/core/table/source/fallback_table_read.h" -#include "paimon/core/table/source/key_value_table_read.h" -#include "paimon/core/table/system/system_table.h" -#include "paimon/core/utils/branch_manager.h" -#include "paimon/core/utils/file_store_path_factory.h" -#include "paimon/defs.h" -#include "paimon/format/file_format.h" -#include "paimon/read_context.h" -#include "paimon/status.h" -#include "paimon/table/format/format_table.h" - namespace paimon { -class DataSplit; -class Executor; -class MemoryPool; - -namespace { - -/// @param loaded_schema The schema of `branch`, when the caller already read it, or null when it -/// has to be read here: `TableRead::Create()` reads one to dispatch on the table type, and -/// reading it again would cost a second listing and read. -Result> CreateInternalReadContext( - const std::shared_ptr& context, const std::string& branch, - const std::shared_ptr& loaded_schema) { - std::map tmp_options = context->GetOptions(); - std::shared_ptr table_schema; - const auto& specific_table_schema = context->GetSpecificTableSchema(); - if (loaded_schema != nullptr) { - table_schema = loaded_schema; - } else if (branch == BranchManager::DEFAULT_MAIN_BRANCH && specific_table_schema) { - PAIMON_ASSIGN_OR_RAISE(table_schema, - TableSchema::CreateFromJson(specific_table_schema.value())); - } else { - PAIMON_ASSIGN_OR_RAISE(CoreOptions tmp_core_options, - CoreOptions::FromMap(tmp_options, context->GetSpecificFileSystem(), - context->GetFileSystemSchemeToIdentifierMap())); - SchemaManager schema_manager(tmp_core_options.GetFileSystem(), context->GetPath(), branch); - PAIMON_ASSIGN_OR_RAISE(std::optional> latest_schema, - schema_manager.Latest()); - if (!latest_schema) { - return Status::Invalid(fmt::format("schema file not found in path {}, branch {}", - context->GetPath(), branch)); - } - table_schema = latest_schema.value(); - } - assert(table_schema); - - // merge options - auto options = table_schema->Options(); - for (const auto& [key, value] : tmp_options) { - options[key] = value; - } - if (branch != BranchManager::DEFAULT_MAIN_BRANCH) { - options[Options::BRANCH] = branch; - } - return InternalReadContext::Create(context, table_schema, options); -} - -Result> CreateTableRead( - const std::shared_ptr& internal_context, - const std::shared_ptr& memory_pool, const std::shared_ptr& executor) { - const auto& core_options = internal_context->GetCoreOptions(); - const auto& table_schema = internal_context->GetTableSchema(); - if (internal_context->GetRealtimeContext() && !core_options.RealtimeEnabled()) { - return Status::Invalid("real-time read requires realtime.enabled=true"); - } - PAIMON_RETURN_NOT_OK(SchemaValidation::ValidateRealtime(*table_schema, core_options)); - auto arrow_schema = DataField::ConvertDataFieldsToArrowSchema(table_schema->Fields()); - PAIMON_ASSIGN_OR_RAISE(std::vector external_paths, - core_options.CreateExternalPaths()); - PAIMON_ASSIGN_OR_RAISE(std::optional global_index_external_path, - core_options.CreateGlobalIndexExternalPath()); - - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr path_factory, - FileStorePathFactory::Create( - internal_context->GetPath(), arrow_schema, table_schema->PartitionKeys(), - core_options.GetPartitionDefaultName(), core_options.GetFileFormat()->Identifier(), - core_options.DataFilePrefix(), core_options.LegacyPartitionNameEnabled(), - external_paths, global_index_external_path, core_options.IndexFileInDataFileDir(), - memory_pool)); - - if (internal_context->GetPrimaryKeys().empty()) { - return std::make_unique(path_factory, internal_context, memory_pool, - executor); - } - return KeyValueTableRead::Create(path_factory, internal_context, memory_pool, executor); -} - -Result> NewDataTableRead( - const std::shared_ptr& context, - const std::shared_ptr& loaded_schema) { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr internal_context, - CreateInternalReadContext(context, context->GetBranch(), loaded_schema)); - - PAIMON_ASSIGN_OR_RAISE( - std::unique_ptr table_read, - CreateTableRead(internal_context, context->GetMemoryPool(), context->GetExecutor())); - - std::optional scan_fallback_branch = - internal_context->GetCoreOptions().GetScanFallbackBranch(); - if (!scan_fallback_branch || - StringUtils::IsNullOrWhitespaceOnly(scan_fallback_branch.value())) { - return std::move(table_read); - } - - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr fallback_context, - CreateInternalReadContext(context, /*branch=*/scan_fallback_branch.value(), - /*loaded_schema=*/nullptr)); - - PAIMON_ASSIGN_OR_RAISE( - std::unique_ptr fallback_table_read, - CreateTableRead(fallback_context, context->GetMemoryPool(), context->GetExecutor())); - const std::shared_ptr& arrow_pool = internal_context->GetArrowMemoryPool(); - return std::make_unique(std::move(table_read), - std::move(fallback_table_read), arrow_pool); -} - -/// Maps a `ReadContext` onto `FormatTableRead`, which reads everything it needs out of the context -/// itself: the columns, the predicate, the pool and executor, and what a file is opened with. It -/// refuses by name what a format table cannot honour. -Result> NewFormatTableRead(const std::shared_ptr& table, - const std::shared_ptr& context) { - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read, - FormatTableRead::Create(table, context)); - return std::unique_ptr(std::move(read)); -} - -} // namespace - -Result> TableRead::Create(std::unique_ptr ctx) { - std::shared_ptr context = std::move(ctx); - if (context == nullptr) { - return Status::Invalid("read context is null pointer"); - } - if (context->GetMemoryPool() == nullptr) { - return Status::Invalid("memory pool is null pointer"); - } - if (context->GetExecutor() == nullptr) { - return Status::Invalid("executor is null pointer"); - } - // A table the caller already loaded says what it is, so nothing is read to find out. - if (context->GetFormatTable() != nullptr) { - if (context->HasVectorOrFullTextSearch()) { - return Status::NotImplemented( - "VectorSearch or FullTextSearch is not supported for format tables"); - } - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr format_table, - FormatTable::Copy(context->GetFormatTable(), context->GetOptions())); - return NewFormatTableRead(format_table, context); - } - PAIMON_ASSIGN_OR_RAISE( - CoreOptions tmp_core_options, - CoreOptions::FromMap(context->GetOptions(), context->GetSpecificFileSystem(), - context->GetFileSystemSchemeToIdentifierMap())); - PAIMON_ASSIGN_OR_RAISE(std::optional system_table_path, - SystemTableLoader::TryParsePath(context->GetPath())); - if (system_table_path) { - if (context->HasVectorOrFullTextSearch()) { - return Status::NotImplemented( - "VectorSearch or FullTextSearch is not supported for system tables"); - } - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr system_table, - SystemTableLoader::LoadFromPath(tmp_core_options.GetFileSystem(), context->GetPath(), - context->GetOptions())); - return system_table->NewRead(context); - } - - // A format table has no manifest and a file another engine wrote carries no field ids, so - // which files there are and how a row is put back together is its own. How a file is opened - // is not: both paths go through `DataFileReaderFactory`. - std::shared_ptr latest_schema; - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr format_table, - FormatTableLoader::TryLoad(tmp_core_options.GetFileSystem(), context->GetPath(), - context->GetBranch(), context->GetOptions(), - context->GetSpecificTableSchema(), - /*schema_manager=*/nullptr, &latest_schema)); - if (format_table != nullptr) { - if (context->HasVectorOrFullTextSearch()) { - return Status::NotImplemented( - "VectorSearch or FullTextSearch is not supported for format tables"); - } - return NewFormatTableRead(format_table, context); - } - - // With the schema the dispatch already read, so the managed path does not read it again. - return NewDataTableRead(context, latest_schema); -} Result> TableRead::CreateCountReader( const std::vector>& splits) { diff --git a/src/paimon/core/table/source/table_scan.cpp b/src/paimon/core/table/source/table_scan.cpp index 21d338f22..be5d5b955 100644 --- a/src/paimon/core/table/source/table_scan.cpp +++ b/src/paimon/core/table/source/table_scan.cpp @@ -19,228 +19,9 @@ #include "paimon/table/source/table_scan.h" -#include -#include -#include -#include -#include -#include - -#include "fmt/format.h" #include "paimon/common/metrics/metrics_impl.h" -#include "paimon/common/predicate/predicate_validator.h" -#include "paimon/common/types/data_field.h" -#include "paimon/common/utils/fields_comparator.h" -#include "paimon/common/utils/options_utils.h" -#include "paimon/core/core_options.h" -#include "paimon/core/index/index_file_handler.h" -#include "paimon/core/index/pk/primary_key_index_definitions.h" -#include "paimon/core/manifest/index_manifest_file.h" -#include "paimon/core/manifest/manifest_file.h" -#include "paimon/core/manifest/manifest_list.h" -#include "paimon/core/operation/append_only_file_store_scan.h" -#include "paimon/core/operation/data_evolution_file_store_scan.h" -#include "paimon/core/operation/file_store_scan.h" -#include "paimon/core/operation/key_value_file_store_scan.h" -#include "paimon/core/realtime/realtime_context_impl.h" -#include "paimon/core/schema/schema_manager.h" -#include "paimon/core/schema/schema_validation.h" -#include "paimon/core/schema/table_schema.h" -#include "paimon/core/table/bucket_mode.h" -#include "paimon/core/table/format/format_table_loader.h" -#include "paimon/core/table/format/format_table_scan.h" -#include "paimon/core/table/source/abstract_table_scan.h" -#include "paimon/core/table/source/append_only_split_generator.h" -#include "paimon/core/table/source/data_evolution_batch_scan.h" -#include "paimon/core/table/source/data_evolution_split_generator.h" -#include "paimon/core/table/source/data_table_batch_scan.h" -#include "paimon/core/table/source/data_table_stream_scan.h" -#include "paimon/core/table/source/merge_tree_split_generator.h" -#include "paimon/core/table/source/primary_key_index_batch_scan.h" -#include "paimon/core/table/source/read_optimized_scan_options.h" -#include "paimon/core/table/source/realtime_table_scan.h" -#include "paimon/core/table/source/snapshot/snapshot_reader.h" -#include "paimon/core/table/source/split_generator.h" -#include "paimon/core/table/system/system_table.h" -#include "paimon/core/utils/branch_manager.h" -#include "paimon/core/utils/field_mapping.h" -#include "paimon/core/utils/file_store_path_factory.h" -#include "paimon/core/utils/index_file_path_factories.h" -#include "paimon/core/utils/primary_key_table_utils.h" -#include "paimon/core/utils/snapshot_manager.h" -#include "paimon/format/file_format.h" -#include "paimon/realtime/realtime_context.h" -#include "paimon/result.h" -#include "paimon/scan_context.h" -#include "paimon/status.h" -#include "paimon/table/format/format_table.h" - -namespace arrow { -class Schema; -} // namespace arrow namespace paimon { -class Executor; -class MemoryPool; - -namespace { - -class TableScanImpl { - public: - static Result> CreateFileStoreScan( - const std::shared_ptr& path_factory, - const std::shared_ptr& arrow_schema, - const std::shared_ptr& table_schema, const CoreOptions& core_options, - const std::shared_ptr& executor, const std::shared_ptr& memory_pool, - const ScanContext* context) { - auto fs = core_options.GetFileSystem(); - auto manifest_file_format = core_options.GetManifestFormat(); - std::string branch = BranchManager::NormalizeBranch(core_options.GetBranch()); - auto snapshot_manager = std::make_shared(fs, context->GetPath(), branch); - // TODO(liancheng.lsz): support fallback branch in scan - auto schema_manager = std::make_shared(fs, context->GetPath(), branch); - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr manifest_list, - ManifestList::Create(fs, manifest_file_format, core_options.GetManifestCompression(), - path_factory, core_options.GetCache(), memory_pool)); - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr partition_schema, - FieldMapping::GetPartitionSchema(arrow_schema, table_schema->PartitionKeys())); - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr manifest_file, - ManifestFile::Create(fs, manifest_file_format, core_options.GetManifestCompression(), - path_factory, core_options.GetManifestTargetFileSize(), - memory_pool, core_options, partition_schema)); - std::unique_ptr scan; - if (table_schema->PrimaryKeys().empty()) { - if (core_options.DataEvolutionEnabled()) { - PAIMON_ASSIGN_OR_RAISE( - scan, DataEvolutionFileStoreScan::Create( - snapshot_manager, schema_manager, manifest_list, manifest_file, - table_schema, arrow_schema, context->GetScanFilters(), core_options, - executor, memory_pool)); - } else { - PAIMON_ASSIGN_OR_RAISE( - scan, AppendOnlyFileStoreScan::Create( - snapshot_manager, schema_manager, manifest_list, manifest_file, - table_schema, arrow_schema, context->GetScanFilters(), core_options, - executor, memory_pool)); - } - } else { - PAIMON_ASSIGN_OR_RAISE( - scan, KeyValueFileStoreScan::Create(snapshot_manager, schema_manager, manifest_list, - manifest_file, table_schema, arrow_schema, - context->GetScanFilters(), core_options, - executor, memory_pool)); - } - return WithTablePath(std::move(scan), context); - } - - static std::unique_ptr WithTablePath(std::unique_ptr&& scan, - const ScanContext* context) { - scan->WithTablePath(context->GetPath()); - return std::move(scan); - } - - static Result> CreateSplitGenerator( - const std::shared_ptr& table_schema, const CoreOptions& core_options, - const ScanContext* context) { - auto source_split_target_size = core_options.GetSourceSplitTargetSize(); - auto source_split_open_file_cost = core_options.GetSourceSplitOpenFileCost(); - if (table_schema->PrimaryKeys().empty()) { - if (core_options.DataEvolutionEnabled()) { - return std::make_unique( - source_split_target_size, source_split_open_file_cost, - core_options.BlobSplitByFileSize()); - } - BucketMode bucket_mode = (core_options.GetBucket() == -1 ? BucketMode::BUCKET_UNAWARE - : BucketMode::HASH_FIXED); - return std::make_unique( - source_split_target_size, source_split_open_file_cost, bucket_mode); - } else { - // TODO(liancheng.lsz): support evolution - PAIMON_ASSIGN_OR_RAISE(std::vector trimmed_primary_keys, - table_schema->TrimmedPrimaryKeys()); - PAIMON_ASSIGN_OR_RAISE(std::vector trimmed_pk_fields, - table_schema->GetFields(trimmed_primary_keys)); - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr key_comparator, - FieldsComparator::Create(trimmed_pk_fields, /*is_ascending_order=*/true)); - return std::make_unique( - source_split_target_size, source_split_open_file_cost, - core_options.DeletionVectorsEnabled(), core_options.GetMergeEngine(), - key_comparator); - } - } - - static Result> CreateIndexFileHandler( - const CoreOptions& core_options, const std::shared_ptr& path_factory, - const std::shared_ptr& memory_pool) { - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr index_manifest_file, - IndexManifestFile::Create( - core_options.GetFileSystem(), core_options.GetManifestFormat(), - core_options.GetManifestCompression(), path_factory, - core_options.GetBucket(), memory_pool, core_options)); - return std::make_unique( - core_options.GetFileSystem(), std::move(index_manifest_file), - std::make_shared(path_factory), - core_options.DeletionVectorsBitmap64(), memory_pool); - } -}; - -/// @param loaded_schema The table's schema, when the caller already read it, or null when it has -/// to be read here. `TableScan::Create()` reads one to dispatch on its table type. -Result> NewDataTableScan( - const std::shared_ptr& context, const std::shared_ptr& loaded_schema); - -/// Maps a `ScanContext` onto `FormatTableScan`, which takes one partition filter and a limit and -/// nothing else. -/// -/// A format table has no buckets, so a bucket filter has nothing to act on. A predicate is -/// refused too, which is narrower than Java: `FormatReadBuilder.newScan()` hands the partition -/// half of one to the scan to prune directories with, which this does not do yet. -Result> NewFormatTableScan(const std::shared_ptr& table, - const std::shared_ptr& context) { - // Anything a `ScanContext` carries that a format table cannot honour is refused rather than - // silently dropped. - if (context->IsStreamingMode()) { - return Status::NotImplemented( - "a format table has no snapshots, so there is nothing for a streaming scan to follow"); - } - if (context->GetRealtimeContext() != nullptr) { - return Status::NotImplemented( - "a format table has no real-time store to union with what is on disk"); - } - if (context->GetGlobalIndexResult() != nullptr) { - return Status::NotImplemented("a format table carries no global index"); - } - std::map partition_filter; - std::shared_ptr filters = context->GetScanFilters(); - if (filters != nullptr) { - if (filters->GetPredicate() != nullptr) { - return Status::NotImplemented( - "a format table scan does not take a predicate: its files carry no statistics to " - "skip by, and pruning by the partition columns a predicate names is not " - "implemented yet; give the partition values as a partition filter instead"); - } - if (filters->GetBucketFilter()) { - return Status::NotImplemented("a format table has no buckets to filter by"); - } - const std::vector>& partition_filters = - filters->GetPartitionFilters(); - if (partition_filters.size() > 1) { - return Status::NotImplemented("a format table scan takes at most one partition filter"); - } - if (partition_filters.size() == 1) { - partition_filter = partition_filters.front(); - } - } - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan, - FormatTableScan::Create(table, partition_filter, context->GetLimit())); - return std::unique_ptr(std::move(scan)); -} - -} // namespace std::shared_ptr TableScan::GetMetrics() const { return std::make_shared(); @@ -252,221 +33,4 @@ Result>> TableScan::ListPartition return Status::NotImplemented("this table does not list its partitions"); } -Result> TableScan::Create(std::unique_ptr context) { - if (context == nullptr) { - return Status::Invalid("scan context is null pointer"); - } - if (context->GetMemoryPool() == nullptr) { - return Status::Invalid("memory pool is null pointer"); - } - if (context->GetExecutor() == nullptr) { - return Status::Invalid("executor is null pointer"); - } - - std::shared_ptr shared_context = std::move(context); - // A table the caller already loaded says what it is, so nothing is read to find out. - if (shared_context->GetFormatTable() != nullptr) { - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr format_table, - FormatTable::Copy(shared_context->GetFormatTable(), shared_context->GetOptions())); - return NewFormatTableScan(format_table, shared_context); - } - // load schema - PAIMON_ASSIGN_OR_RAISE(CoreOptions tmp_options, - CoreOptions::FromMap(shared_context->GetOptions(), - shared_context->GetSpecificFileSystem(), {})); - PAIMON_ASSIGN_OR_RAISE(std::optional system_table_path, - SystemTableLoader::TryParsePath(shared_context->GetPath())); - if (system_table_path) { - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr system_table, - SystemTableLoader::LoadFromPath(tmp_options.GetFileSystem(), shared_context->GetPath(), - shared_context->GetOptions())); - return system_table->NewScan(shared_context); - } - // A format table is planned by listing directories, so it never reaches the manifest path - // below. One `TableScan` interface serves both, as Java Paimon serves both through one - // `ReadBuilder`. - std::shared_ptr latest_schema; - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr format_table, - FormatTableLoader::TryLoad(tmp_options.GetFileSystem(), shared_context->GetPath(), - BranchManager::NormalizeBranch(tmp_options.GetBranch()), - shared_context->GetOptions(), - shared_context->GetSpecificTableSchema(), - /*schema_manager=*/nullptr, &latest_schema)); - if (format_table != nullptr) { - return NewFormatTableScan(format_table, shared_context); - } - // With the schema the dispatch already read, so the managed path does not read it again. - return NewDataTableScan(shared_context, latest_schema); -} - -namespace { - -Status ValidateRealtimeScan(const TableSchema& table_schema, const CoreOptions& core_options, - const ScanContext& context, bool read_optimized) { - if (!context.GetRealtimeContext()) { - return Status::OK(); - } - if (!core_options.RealtimeEnabled()) { - return Status::Invalid("real-time scan requires realtime.enabled=true"); - } - if (core_options.GetBucket() <= 0) { - return Status::Invalid("real-time union read requires fixed bucket mode"); - } - if (core_options.DataEvolutionEnabled()) { - return Status::Invalid("real-time union read does not support data evolution"); - } - if (!table_schema.PrimaryKeys().empty()) { - if (read_optimized) { - return Status::NotImplemented( - "PK real-time union read does not support read-optimized scans"); - } - PAIMON_RETURN_NOT_OK( - PrimaryKeyTableUtils::ValidateRealtimeOptions(core_options, table_schema)); - } - if (context.IsStreamingMode()) { - return Status::Invalid("real-time union read currently supports batch scans only"); - } - if (context.GetLimit()) { - return Status::Invalid("real-time union read does not support scan limit pushdown"); - } - if (context.GetGlobalIndexResult()) { - return Status::Invalid("real-time union read does not support global index splits"); - } - StartupMode startup_mode = core_options.GetStartupMode(); - const bool specified_snapshot = - startup_mode == StartupMode::FromSnapshot() && core_options.GetScanSnapshotId().has_value(); - if (!(startup_mode == StartupMode::LatestFull() || startup_mode == StartupMode::Latest() || - specified_snapshot)) { - return Status::Invalid( - "real-time union read requires the latest snapshot or an explicit snapshot id"); - } - return Status::OK(); -} - -Result> NewDataTableScan( - const std::shared_ptr& context, - const std::shared_ptr& loaded_schema) { - PAIMON_ASSIGN_OR_RAISE( - CoreOptions tmp_options, - CoreOptions::FromMap(context->GetOptions(), context->GetSpecificFileSystem(), {})); - std::string branch = BranchManager::NormalizeBranch(tmp_options.GetBranch()); - std::shared_ptr table_schema; - const auto& specific_table_schema = context->GetSpecificTableSchema(); - if (loaded_schema != nullptr) { - table_schema = loaded_schema; - } else if (branch == BranchManager::DEFAULT_MAIN_BRANCH && specific_table_schema) { - PAIMON_ASSIGN_OR_RAISE(table_schema, - TableSchema::CreateFromJson(specific_table_schema.value())); - } else { - SchemaManager schema_manager(tmp_options.GetFileSystem(), context->GetPath(), branch); - PAIMON_ASSIGN_OR_RAISE(std::optional> latest_table_schema, - schema_manager.Latest()); - if (latest_table_schema == std::nullopt) { - return Status::Invalid("not found latest schema"); - } - table_schema = latest_table_schema.value(); - } - PAIMON_ASSIGN_OR_RAISE(bool read_optimized, - OptionsUtils::GetValueFromMap(context->GetOptions(), - kReadOptimizedScanOption, false)); - // merge options - auto options = table_schema->Options(); - for (const auto& [key, value] : context->GetOptions()) { - options[key] = value; - } - PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, - CoreOptions::FromMap(options, context->GetSpecificFileSystem(), {})); - core_options.WithCache(context->GetCache()); - - PAIMON_RETURN_NOT_OK(SchemaValidation::ValidateRealtime(*table_schema, core_options)); - - PAIMON_RETURN_NOT_OK( - ValidateRealtimeScan(*table_schema, core_options, *context, read_optimized)); - // validate options - if (core_options.GetBucket() == -1) { - if (!table_schema->PrimaryKeys().empty()) { - return Status::NotImplemented(fmt::format( - "do not support pk table bucket={} in scan process", core_options.GetBucket())); - } - } else if (core_options.GetBucket() < 1 && - !SchemaValidation::IsPostponeBucketTable(*table_schema, core_options.GetBucket())) { - return Status::Invalid( - fmt::format("do not support bucket={} in scan process", core_options.GetBucket())); - } - - // validate schema and scan filter - auto arrow_schema = DataField::ConvertDataFieldsToArrowSchema(table_schema->Fields()); - if (context->GetScanFilters() && context->GetScanFilters()->GetPredicate()) { - PAIMON_RETURN_NOT_OK(PredicateValidator::ValidatePredicateWithSchema( - *arrow_schema, context->GetScanFilters()->GetPredicate(), - /*validate_field_idx=*/false)); - } - PAIMON_ASSIGN_OR_RAISE(std::vector external_paths, - core_options.CreateExternalPaths()); - PAIMON_ASSIGN_OR_RAISE(std::optional global_index_external_path, - core_options.CreateGlobalIndexExternalPath()); - - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr path_factory, - FileStorePathFactory::Create( - context->GetPath(), arrow_schema, table_schema->PartitionKeys(), - core_options.GetPartitionDefaultName(), core_options.GetFileFormat()->Identifier(), - core_options.DataFilePrefix(), core_options.LegacyPartitionNameEnabled(), - external_paths, global_index_external_path, core_options.IndexFileInDataFileDir(), - context->GetMemoryPool())); - - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr file_store_scan, - TableScanImpl::CreateFileStoreScan( - path_factory, arrow_schema, table_schema, core_options, - context->GetExecutor(), context->GetMemoryPool(), context.get())); - PAIMON_ASSIGN_OR_RAISE( - std::unique_ptr split_generator, - TableScanImpl::CreateSplitGenerator(table_schema, core_options, context.get())); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr index_file_handler, - TableScanImpl::CreateIndexFileHandler(core_options, path_factory, - context->GetMemoryPool())); - auto snapshot_reader = std::make_shared( - file_store_scan, path_factory, std::move(split_generator), std::move(index_file_handler)); - const bool pk_table = !table_schema->PrimaryKeys().empty(); - if (read_optimized && pk_table && context->IsStreamingMode()) { - return Status::NotImplemented( - "read-optimized system table does not support streaming scan for primary key table"); - } - if (context->IsStreamingMode()) { - return std::make_unique(core_options, snapshot_reader); - } - auto batch_scan = std::make_unique( - /*pk_table=*/pk_table, core_options, snapshot_reader, read_optimized, context->GetLimit(), - /*realtime_pk_scan=*/pk_table && context->GetRealtimeContext() != nullptr); - if (context->GetRealtimeContext()) { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr realtime_context, - RealtimeContextImpl::Cast(context->GetRealtimeContext())); - return std::make_unique( - std::move(batch_scan), pk_table, realtime_context, path_factory, - snapshot_reader->GetSnapshotManager(), core_options.GetFileSystem(), - context->GetScanFilters(), core_options.GetRealtimeReadViewTtlMillis()); - } - if (core_options.DataEvolutionEnabled()) { - return std::make_unique( - context->GetPath(), snapshot_reader, std::move(batch_scan), table_schema, - context->GetGlobalIndexResult(), core_options, context->GetMemoryPool(), - context->GetExecutor()); - } - if (pk_table && !read_optimized && core_options.GlobalIndexEnabled()) { - PAIMON_ASSIGN_OR_RAISE(PrimaryKeyIndexDefinitions definitions, - PrimaryKeyIndexDefinitions::Create(*table_schema)); - if (!definitions.ScalarDefinitions().empty()) { - return PrimaryKeyIndexBatchScan::Create(snapshot_reader, std::move(batch_scan), - table_schema, path_factory, core_options, - context->GetMemoryPool()); - } - } - return batch_scan; -} - -} // namespace - } // namespace paimon diff --git a/src/paimon/core/table/table.cpp b/src/paimon/core/table/table.cpp index bc1e64fbc..479f1037e 100644 --- a/src/paimon/core/table/table.cpp +++ b/src/paimon/core/table/table.cpp @@ -19,27 +19,192 @@ #include "paimon/catalog/table.h" +#include +#include +#include #include #include #include #include +#include +#include "arrow/c/bridge.h" #include "fmt/format.h" +#include "paimon/catalog/catalog.h" #include "paimon/commit_context.h" +#include "paimon/common/global_index/btree/btree_defs.h" +#include "paimon/common/predicate/predicate_validator.h" +#include "paimon/common/types/data_field.h" +#include "paimon/common/utils/arrow/status_utils.h" +#include "paimon/common/utils/binary_row_partition_computer.h" #include "paimon/common/utils/checked_cast.h" +#include "paimon/common/utils/fields_comparator.h" +#include "paimon/common/utils/options_utils.h" +#include "paimon/common/utils/string_utils.h" +#include "paimon/core/catalog/catalog_snapshot_commit.h" #include "paimon/core/catalog/catalog_utils.h" +#include "paimon/core/catalog/renaming_snapshot_commit.h" +#include "paimon/core/catalog/snapshot_commit.h" +#include "paimon/core/catalog/version_managed_catalog.h" +#include "paimon/core/core_options.h" +#include "paimon/core/disk/io_manager.h" +#include "paimon/core/index/index_file_handler.h" +#include "paimon/core/index/pk/bucketed_primary_key_index_maintainer.h" +#include "paimon/core/index/pk/primary_key_index_definitions.h" +#include "paimon/core/manifest/index_manifest_entry.h" +#include "paimon/core/manifest/index_manifest_file.h" +#include "paimon/core/mergetree/compact/lookup_merge_function.h" +#include "paimon/core/mergetree/compact/merge_function.h" +#include "paimon/core/mergetree/compact/reducer_merge_function_wrapper.h" +#include "paimon/core/operation/append_only_file_store_scan.h" +#include "paimon/core/operation/append_only_file_store_write.h" +#include "paimon/core/operation/commit/realtime_commit_properties.h" +#include "paimon/core/operation/data_evolution_file_store_scan.h" +#include "paimon/core/operation/expire_snapshots.h" +#include "paimon/core/operation/file_store_commit_impl.h" +#include "paimon/core/operation/file_store_scan.h" +#include "paimon/core/operation/internal_read_context.h" +#include "paimon/core/operation/key_value_file_store_scan.h" +#include "paimon/core/operation/key_value_file_store_write.h" +#include "paimon/core/options/merge_engine.h" +#include "paimon/core/postpone/postpone_bucket_file_store_write.h" +#include "paimon/core/realtime/realtime_context_impl.h" +#include "paimon/core/realtime/realtime_schema_layout.h" #include "paimon/core/schema/schema_manager.h" +#include "paimon/core/schema/schema_validation.h" #include "paimon/core/schema/table_schema.h" +#include "paimon/core/table/bucket_mode.h" +#include "paimon/core/table/format/format_table_file_store_commit.h" +#include "paimon/core/table/format/format_table_file_store_write.h" +#include "paimon/core/table/format/format_table_loader.h" +#include "paimon/core/table/format/format_table_read.h" +#include "paimon/core/table/format/format_table_scan.h" +#include "paimon/core/table/source/abstract_table_scan.h" +#include "paimon/core/table/source/append_only_split_generator.h" +#include "paimon/core/table/source/append_only_table_read.h" +#include "paimon/core/table/source/data_evolution_batch_scan.h" +#include "paimon/core/table/source/data_evolution_split_generator.h" +#include "paimon/core/table/source/data_table_batch_scan.h" +#include "paimon/core/table/source/data_table_stream_scan.h" +#include "paimon/core/table/source/fallback_table_read.h" +#include "paimon/core/table/source/key_value_table_read.h" +#include "paimon/core/table/source/merge_tree_split_generator.h" +#include "paimon/core/table/source/primary_key_index_batch_scan.h" +#include "paimon/core/table/source/read_optimized_scan_options.h" +#include "paimon/core/table/source/realtime_table_scan.h" +#include "paimon/core/table/source/snapshot/snapshot_reader.h" +#include "paimon/core/table/source/split_generator.h" +#include "paimon/core/table/system/system_table.h" +#include "paimon/core/utils/branch_manager.h" +#include "paimon/core/utils/field_mapping.h" +#include "paimon/core/utils/file_store_path_factory.h" +#include "paimon/core/utils/index_file_path_factories.h" +#include "paimon/core/utils/primary_key_table_utils.h" +#include "paimon/core/utils/snapshot_manager.h" +#include "paimon/defs.h" #include "paimon/file_store_commit.h" #include "paimon/file_store_write.h" +#include "paimon/format/file_format.h" #include "paimon/fs/file_system.h" #include "paimon/read_context.h" +#include "paimon/realtime/realtime_context.h" +#include "paimon/result.h" #include "paimon/scan_context.h" +#include "paimon/schema/schema.h" +#include "paimon/status.h" +#include "paimon/table/format/format_table.h" #include "paimon/table/source/table_read.h" #include "paimon/table/source/table_scan.h" #include "paimon/write_context.h" +namespace arrow { +class Schema; +} // namespace arrow + namespace paimon { +class DataSplit; +class Executor; +struct KeyValue; +class MemoryPool; +template +class MergeFunctionWrapper; + +namespace { + +Status RestoreRealtimeCommittedProgress(const std::shared_ptr& realtime_context, + const std::shared_ptr& snapshot_manager, + const CoreOptions& options) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr realtime_context_impl, + RealtimeContextImpl::Cast(realtime_context)); + PAIMON_RETURN_NOT_OK(realtime_context_impl->CheckUsable()); + PAIMON_ASSIGN_OR_RAISE(std::optional latest_snapshot, + snapshot_manager->LatestSnapshot()); + if (latest_snapshot) { + PAIMON_ASSIGN_OR_RAISE( + RealtimeOffsetMap realtime_committed_offsets, + RealtimeCommitProperties::ReadOffsets(latest_snapshot, options.GetFileSystem())); + PAIMON_RETURN_NOT_OK(realtime_context_impl->AdvanceCommittedProgress( + latest_snapshot->Id(), realtime_committed_offsets, /*committed_data_files=*/{})); + } + return Status::OK(); +} + +/// Maps a `WriteContext` onto `FormatTableFileStoreWrite`, refusing by name what a format table +/// cannot honour rather than silently dropping it. +Result> NewFormatTableWrite( + const std::shared_ptr& table, const WriteContext& ctx) { + if (ctx.GetCatalog() != nullptr) { + return Status::Invalid( + "WithCatalog() requires a native table; use WriteContextBuilder(FormatTable) for a " + "format table"); + } + if (ctx.IsStreamingMode()) { + return Status::NotImplemented( + "a format table has no snapshots, so there is nothing a streaming write could " + "commit against"); + } + if (ctx.GetRealtimeContext() != nullptr) { + return Status::NotImplemented("a format table has no real-time store to write into"); + } + if (!ctx.GetWriteSchema().empty()) { + return Status::NotImplemented( + "a format table write takes the table's own columns; a write schema naming a " + "subset of them is not supported yet"); + } + // A write id prefixes a postpone-bucket writer's files so that one compaction reader can + // put them back in order. A format table has no buckets, so it would identify nothing. + if (ctx.GetWriteId().has_value()) { + return Status::NotImplemented( + "a format table has no buckets, so a write id would name nothing"); + } + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr format_write, + FormatTableFileStoreWrite::Create(table, ctx.GetMemoryPool())); + return std::unique_ptr(std::move(format_write)); +} + +Result HasHistoricalPrimaryKeyBTreeDefinition( + const std::shared_ptr& schema_manager, int64_t current_schema_id) { + if (current_schema_id <= TableSchema::FIRST_SCHEMA_ID) { + return false; + } + PAIMON_ASSIGN_OR_RAISE(std::vector schema_ids, schema_manager->ListAllIds()); + for (int64_t schema_id : schema_ids) { + if (schema_id >= current_schema_id) { + continue; + } + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr historical_schema, + schema_manager->ReadSchema(schema_id)); + const auto& historical_options = historical_schema->Options(); + auto option = historical_options.find(Options::PK_BTREE_INDEX_COLUMNS); + if (option != historical_options.end() && + !StringUtils::IsNullOrWhitespaceOnly(option->second)) { + return true; + } + } + return false; +} + +} // namespace Table::Table(const std::shared_ptr& file_system, const std::string& root_path, const std::shared_ptr& schema, const Identifier& identifier, @@ -99,4 +264,1183 @@ Result> Table::NewCommit( return FileStoreCommit::Create(std::move(context)); } +Result> FileStoreWrite::Create(std::unique_ptr ctx) { + if (ctx == nullptr) { + return Status::Invalid("write context is null pointer"); + } + if (ctx->GetMemoryPool() == nullptr) { + return Status::Invalid("memory pool is null pointer"); + } + if (ctx->GetExecutor() == nullptr) { + return Status::Invalid("executor is null pointer"); + } + + // A table the caller already loaded says what it is, so nothing is read to find out. + if (ctx->GetFormatTable() != nullptr) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr given_table, + FormatTable::Copy(ctx->GetFormatTable(), ctx->GetOptions())); + return NewFormatTableWrite(given_table, *ctx); + } + + PAIMON_RETURN_NOT_OK(CheckVersionManagementImplemented(ctx->GetCatalog())); + std::shared_ptr specific_fs = ctx->GetSpecificFileSystem(); + if (specific_fs == nullptr && ctx->GetFileSystemSchemeToIdentifierMap().empty() && + ctx->GetCatalog() != nullptr) { + if (!ctx->GetIdentifier()) { + return Status::Invalid("a catalog write requires a table identifier"); + } + // A catalog issuing per-table temporary credentials only hands them out through + // GetTableFileSystem, so the write uses the credentials of this table. + PAIMON_ASSIGN_OR_RAISE(specific_fs, + ctx->GetCatalog()->GetTableFileSystem(ctx->GetIdentifier().value())); + } + PAIMON_ASSIGN_OR_RAISE(CoreOptions tmp_options, + CoreOptions::FromMap(ctx->GetOptions(), specific_fs, + ctx->GetFileSystemSchemeToIdentifierMap())); + std::string branch = ctx->GetBranch(); + std::optional catalog_identifier; + if (ctx->GetIdentifier()) { + PAIMON_ASSIGN_OR_RAISE( + Identifier branch_identifier, + CatalogUtils::BranchIdentifier(ctx->GetIdentifier().value(), branch)); + catalog_identifier.emplace(std::move(branch_identifier)); + } + // Only the main branch may have a current schema the catalog alone holds. A branch publishes + // its own under `branch/branch-`, which is where the load below and a read of the branch + // both read it, so a catalog answering for no branch schema does not fail a write to one. + std::optional catalog_table_schema; + if (ctx->GetCatalog() != nullptr) { + if (!catalog_identifier) { + return Status::Invalid("a catalog write requires a table identifier"); + } + if (BranchManager::IsMainBranch(branch)) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, + ctx->GetCatalog()->LoadTableSchema(catalog_identifier.value())); + PAIMON_ASSIGN_OR_RAISE(catalog_table_schema, schema->GetJsonSchema()); + } + } + // A format table writes plain data files into a directory, so it never reaches the manifest + // path below. + auto schema_manager = + std::make_shared(tmp_options.GetFileSystem(), ctx->GetRootPath(), branch); + std::shared_ptr latest_schema; + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr format_table, + FormatTableLoader::TryLoad(tmp_options.GetFileSystem(), ctx->GetRootPath(), branch, + ctx->GetOptions(), catalog_table_schema, schema_manager.get(), + &latest_schema)); + if (format_table != nullptr) { + return NewFormatTableWrite(format_table, *ctx); + } + // The schema the dispatch above already read through `schema_manager`, rather than a second + // read of the same file. + if (latest_schema == nullptr) { + return Status::Invalid(fmt::format("not found latest schema in branch {}", branch)); + } + const std::shared_ptr& schema = latest_schema; + auto arrow_schema = DataField::ConvertDataFieldsToArrowSchema(schema->Fields()); + auto opts = schema->Options(); + for (const auto& [key, value] : ctx->GetOptions()) { + opts[key] = value; + } + // The metadata paths above are built for this branch, so what reads through these options, + // whose cache keys are scoped by it, has to read the same one. + opts[Options::BRANCH] = branch; + PAIMON_ASSIGN_OR_RAISE( + CoreOptions options, + CoreOptions::FromMap(opts, specific_fs, ctx->GetFileSystemSchemeToIdentifierMap())); + PAIMON_RETURN_NOT_OK(SchemaValidation::ValidateRealtime(*schema, options)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr partition_schema, + FieldMapping::GetPartitionSchema(arrow_schema, schema->PartitionKeys())); + + PAIMON_ASSIGN_OR_RAISE(std::vector external_paths, options.CreateExternalPaths()); + PAIMON_ASSIGN_OR_RAISE(std::optional global_index_external_path, + options.CreateGlobalIndexExternalPath()); + + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr file_store_path_factory, + FileStorePathFactory::Create(ctx->GetRootPath(), arrow_schema, schema->PartitionKeys(), + options.GetPartitionDefaultName(), + options.GetWriteFileFormat(/*level=*/0)->Identifier(), + options.DataFilePrefix(), options.LegacyPartitionNameEnabled(), + external_paths, global_index_external_path, + options.IndexFileInDataFileDir(), ctx->GetMemoryPool())); + auto snapshot_manager = + std::make_shared(options.GetFileSystem(), ctx->GetRootPath(), branch); + if (VersionManagedCatalog* versioned = AsVersionManaged(ctx->GetCatalog())) { + std::shared_ptr catalog = ctx->GetCatalog(); + Identifier identifier = catalog_identifier.value(); + // Keep the catalog alive while the writer uses its snapshot loader. + snapshot_manager->SetSnapshotLoader( + [catalog, versioned, identifier]() -> Result> { + return versioned->LoadSnapshot(identifier); + }); + } + std::shared_ptr io_manager; + const auto& io_temp_dir = ctx->GetTempDirectory(); + if (!io_temp_dir.empty()) { + io_manager = std::make_shared(io_temp_dir, options.GetFileSystem()); + } + + bool ignore_previous_files = ctx->IgnorePreviousFiles(); + if (ctx->GetRealtimeContext() && !options.RealtimeEnabled()) { + return Status::Invalid("real-time write requires realtime.enabled=true"); + } + if (ctx->GetRealtimeContext()) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr realtime_context_impl, + RealtimeContextImpl::Cast(ctx->GetRealtimeContext())); + PAIMON_RETURN_NOT_OK(realtime_context_impl->CheckUsable()); + } + if (schema->PrimaryKeys().empty()) { + // append table + const bool realtime_deduplicate = !options.GetRealtimeDeduplicateKeyFields().empty(); + bool need_dv_maintainer_factory = options.DeletionVectorsEnabled(); + if (options.GetBucket() == -1) { + need_dv_maintainer_factory = false; + ignore_previous_files = true; + } else if (options.GetBucket() <= 0) { + return Status::Invalid( + fmt::format("not support bucket {} in append table", options.GetBucket())); + } + if (ctx->GetRealtimeContext()) { + if (!ctx->GetWriteSchema().empty()) { + return Status::Invalid("real-time append write does not support a custom schema"); + } + if (!realtime_deduplicate) { + PAIMON_RETURN_NOT_OK(RestoreRealtimeCommittedProgress(ctx->GetRealtimeContext(), + snapshot_manager, options)); + } + } else if (realtime_deduplicate) { + return Status::Invalid("real-time deduplicate write requires a real-time context"); + } + std::shared_ptr write_schema = arrow_schema; + const auto& write_field_names = ctx->GetWriteSchema(); + if (!write_field_names.empty()) { + arrow::FieldVector write_fields; + write_fields.reserve(write_field_names.size()); + for (const auto& field_name : write_field_names) { + auto field = arrow_schema->GetFieldByName(field_name); + if (!field) { + // TODO(xinyu.lxy): support _ROW_ID and _SEQUENCE_NUMBER + return Status::Invalid( + fmt::format("write field {} does not exist in table schema", field_name)); + } + write_fields.push_back(field); + } + write_schema = arrow::schema(write_fields); + } + + std::shared_ptr realtime_schema_layout; + if (ctx->GetRealtimeContext()) { + const RealtimeStoreMode mode = realtime_deduplicate ? RealtimeStoreMode::DEDUPLICATE + : RealtimeStoreMode::APPEND_ONLY; + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr layout, + RealtimeSchemaLayout::Create(mode, write_schema)); + realtime_schema_layout = std::move(layout); + } + + std::shared_ptr dv_maintainer_factory; + if (need_dv_maintainer_factory) { + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr index_manifest_file, + IndexManifestFile::Create(options.GetFileSystem(), options.GetManifestFormat(), + options.GetManifestCompression(), file_store_path_factory, + options.GetBucket(), ctx->GetMemoryPool(), options)); + auto index_file_handler = std::make_shared( + options.GetFileSystem(), std::move(index_manifest_file), + std::make_shared(file_store_path_factory), + options.DeletionVectorsBitmap64(), ctx->GetMemoryPool()); + dv_maintainer_factory = + std::make_shared(index_file_handler); + } + + auto file_store_write = std::make_unique( + file_store_path_factory, snapshot_manager, schema_manager, ctx->GetCommitUser(), + ctx->GetRootPath(), schema, arrow_schema, write_schema, realtime_schema_layout, + partition_schema, dv_maintainer_factory, io_manager, options, ignore_previous_files, + ctx->IsStreamingMode(), ctx->IgnoreNumBucketCheck(), ctx->GetRealtimeContext(), + ctx->GetExecutor(), ctx->GetMemoryPool()); + if (ctx->GetRealtimeContext() && realtime_deduplicate) { + PAIMON_ASSIGN_OR_RAISE(std::optional latest_snapshot, + snapshot_manager->LatestSnapshot()); + if (latest_snapshot) { + PAIMON_RETURN_NOT_OK( + file_store_write->RefreshCommittedSnapshot(latest_snapshot->Id())); + } + } + return std::unique_ptr(std::move(file_store_write)); + } else { + // pk table + if (ctx->GetRealtimeContext()) { + PAIMON_RETURN_NOT_OK(PrimaryKeyTableUtils::ValidateRealtimeOptions(options, *schema)); + if (ignore_previous_files) { + return Status::NotImplemented( + "PK realtime requires restore from the latest snapshot"); + } + if (!ctx->GetWriteSchema().empty()) { + return Status::NotImplemented("PK realtime does not support a custom write schema"); + } + PAIMON_RETURN_NOT_OK(RestoreRealtimeCommittedProgress(ctx->GetRealtimeContext(), + snapshot_manager, options)); + } + if (options.GetBucket() == BucketModeDefine::POSTPONE_BUCKET) { + return PostponeBucketFileStoreWrite::Create( + snapshot_manager, schema_manager, ctx->GetCommitUser(), ctx->GetRootPath(), schema, + arrow_schema, partition_schema, io_manager, options, ctx->IsStreamingMode(), + ctx->IgnoreNumBucketCheck(), ctx->GetWriteId(), + ctx->GetFileSystemSchemeToIdentifierMap(), ctx->GetExecutor(), ctx->GetMemoryPool(), + specific_fs); + } + if (options.GetBucket() <= 0) { + return Status::Invalid( + fmt::format("not support bucket {} in key value table", options.GetBucket())); + } + PAIMON_ASSIGN_OR_RAISE(std::vector trimmed_primary_keys, + schema->TrimmedPrimaryKeys()); + PAIMON_ASSIGN_OR_RAISE(std::vector trimmed_primary_key_fields, + schema->GetFields(trimmed_primary_keys)); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr key_comparator, + FieldsComparator::Create(trimmed_primary_key_fields, + options.SequenceFieldSortOrderIsAscending())); + auto primary_keys = schema->PrimaryKeys(); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr merge_function, + PrimaryKeyTableUtils::CreateMergeFunction( + arrow_schema, primary_keys, options, ctx->GetMemoryPool())); + if (options.NeedLookup() && options.GetMergeEngine() != MergeEngine::FIRST_ROW) { + // don't wrap first row, it is already OK + merge_function = std::make_unique(std::move(merge_function)); + } + std::shared_ptr> merge_function_wrapper = + std::make_shared(std::move(merge_function)); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr sequence_fields_comparator, + PrimaryKeyTableUtils::CreateSequenceFieldsComparator(schema->Fields(), options)); + + std::shared_ptr dv_maintainer_factory; + PAIMON_ASSIGN_OR_RAISE(PrimaryKeyIndexDefinitions primary_key_index_definitions, + PrimaryKeyIndexDefinitions::Create(*schema)); + bool has_btree_index = false; + for (const PrimaryKeyIndexDefinition& definition : + primary_key_index_definitions.Definitions()) { + if (definition.GetFamily() == PrimaryKeyIndexDefinition::Family::BTREE) { + has_btree_index = true; + break; + } + } + std::optional latest_snapshot; + bool has_existing_index_manifest = false; + if (!has_btree_index) { + PAIMON_ASSIGN_OR_RAISE( + bool has_historical_btree_index, + HasHistoricalPrimaryKeyBTreeDefinition(schema_manager, schema->Id())); + if (has_historical_btree_index) { + PAIMON_ASSIGN_OR_RAISE(latest_snapshot, snapshot_manager->LatestSnapshot()); + has_existing_index_manifest = + latest_snapshot.has_value() && latest_snapshot->IndexManifest().has_value(); + } + } + std::shared_ptr index_file_handler; + if (options.DeletionVectorsEnabled() || has_btree_index || has_existing_index_manifest) { + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr index_manifest_file, + IndexManifestFile::Create(options.GetFileSystem(), options.GetManifestFormat(), + options.GetManifestCompression(), file_store_path_factory, + options.GetBucket(), ctx->GetMemoryPool(), options)); + index_file_handler = std::make_shared( + options.GetFileSystem(), std::move(index_manifest_file), + std::make_shared(file_store_path_factory), + options.DeletionVectorsBitmap64(), ctx->GetMemoryPool()); + } + bool has_restored_btree_index = false; + if (!has_btree_index && has_existing_index_manifest) { + PAIMON_ASSIGN_OR_RAISE( + std::vector restored_btree_entries, + index_file_handler->Scan( + latest_snapshot.value(), [](const IndexManifestEntry& entry) -> bool { + return entry.index_file->IndexType() == BtreeDefs::kIdentifier && + IndexFileHandler::IsPrimaryKeySourceIndex(*entry.index_file); + })); + has_restored_btree_index = !restored_btree_entries.empty(); + } + if (options.DeletionVectorsEnabled()) { + dv_maintainer_factory = + std::make_shared(index_file_handler); + } + std::shared_ptr + primary_key_index_maintainer_factory; + if (has_btree_index || has_restored_btree_index) { + primary_key_index_maintainer_factory = + std::make_shared( + ctx->GetRootPath(), branch, schema, primary_key_index_definitions.Definitions(), + file_store_path_factory, index_file_handler, options, io_manager, + ctx->EnableMultiThreadSpill(), ctx->GetExecutor(), ctx->GetMemoryPool()); + } + + std::shared_ptr realtime_schema_layout; + if (ctx->GetRealtimeContext()) { + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr layout, + RealtimeSchemaLayout::Create(RealtimeStoreMode::PRIMARY_KEY, arrow_schema)); + realtime_schema_layout = std::move(layout); + } + + return std::make_unique( + file_store_path_factory, snapshot_manager, schema_manager, ctx->GetCommitUser(), + ctx->GetRootPath(), schema, arrow_schema, realtime_schema_layout, partition_schema, + dv_maintainer_factory, primary_key_index_maintainer_factory, io_manager, key_comparator, + sequence_fields_comparator, merge_function_wrapper, options, ignore_previous_files, + ctx->IsStreamingMode(), ctx->IgnoreNumBucketCheck(), ctx->EnableMultiThreadSpill(), + ctx->GetRealtimeContext(), ctx->GetExecutor(), ctx->GetMemoryPool()); + } +} + +namespace { + +class TableScanImpl { + public: + static Result> CreateFileStoreScan( + const std::shared_ptr& path_factory, + const std::shared_ptr& arrow_schema, + const std::shared_ptr& table_schema, const CoreOptions& core_options, + const std::shared_ptr& executor, const std::shared_ptr& memory_pool, + const ScanContext* context) { + auto fs = core_options.GetFileSystem(); + auto manifest_file_format = core_options.GetManifestFormat(); + std::string branch = BranchManager::NormalizeBranch(core_options.GetBranch()); + auto snapshot_manager = std::make_shared(fs, context->GetPath(), branch); + // TODO(liancheng.lsz): support fallback branch in scan + auto schema_manager = std::make_shared(fs, context->GetPath(), branch); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr manifest_list, + ManifestList::Create(fs, manifest_file_format, core_options.GetManifestCompression(), + path_factory, core_options.GetCache(), memory_pool)); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr partition_schema, + FieldMapping::GetPartitionSchema(arrow_schema, table_schema->PartitionKeys())); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr manifest_file, + ManifestFile::Create(fs, manifest_file_format, core_options.GetManifestCompression(), + path_factory, core_options.GetManifestTargetFileSize(), + memory_pool, core_options, partition_schema)); + std::unique_ptr scan; + if (table_schema->PrimaryKeys().empty()) { + if (core_options.DataEvolutionEnabled()) { + PAIMON_ASSIGN_OR_RAISE( + scan, DataEvolutionFileStoreScan::Create( + snapshot_manager, schema_manager, manifest_list, manifest_file, + table_schema, arrow_schema, context->GetScanFilters(), core_options, + executor, memory_pool)); + } else { + PAIMON_ASSIGN_OR_RAISE( + scan, AppendOnlyFileStoreScan::Create( + snapshot_manager, schema_manager, manifest_list, manifest_file, + table_schema, arrow_schema, context->GetScanFilters(), core_options, + executor, memory_pool)); + } + } else { + PAIMON_ASSIGN_OR_RAISE( + scan, KeyValueFileStoreScan::Create(snapshot_manager, schema_manager, manifest_list, + manifest_file, table_schema, arrow_schema, + context->GetScanFilters(), core_options, + executor, memory_pool)); + } + return WithTablePath(std::move(scan), context); + } + + static std::unique_ptr WithTablePath(std::unique_ptr&& scan, + const ScanContext* context) { + scan->WithTablePath(context->GetPath()); + return std::move(scan); + } + + static Result> CreateSplitGenerator( + const std::shared_ptr& table_schema, const CoreOptions& core_options, + const ScanContext* context) { + auto source_split_target_size = core_options.GetSourceSplitTargetSize(); + auto source_split_open_file_cost = core_options.GetSourceSplitOpenFileCost(); + if (table_schema->PrimaryKeys().empty()) { + if (core_options.DataEvolutionEnabled()) { + return std::make_unique( + source_split_target_size, source_split_open_file_cost, + core_options.BlobSplitByFileSize()); + } + BucketMode bucket_mode = (core_options.GetBucket() == -1 ? BucketMode::BUCKET_UNAWARE + : BucketMode::HASH_FIXED); + return std::make_unique( + source_split_target_size, source_split_open_file_cost, bucket_mode); + } else { + // TODO(liancheng.lsz): support evolution + PAIMON_ASSIGN_OR_RAISE(std::vector trimmed_primary_keys, + table_schema->TrimmedPrimaryKeys()); + PAIMON_ASSIGN_OR_RAISE(std::vector trimmed_pk_fields, + table_schema->GetFields(trimmed_primary_keys)); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr key_comparator, + FieldsComparator::Create(trimmed_pk_fields, /*is_ascending_order=*/true)); + return std::make_unique( + source_split_target_size, source_split_open_file_cost, + core_options.DeletionVectorsEnabled(), core_options.GetMergeEngine(), + key_comparator); + } + } + + static Result> CreateIndexFileHandler( + const CoreOptions& core_options, const std::shared_ptr& path_factory, + const std::shared_ptr& memory_pool) { + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr index_manifest_file, + IndexManifestFile::Create( + core_options.GetFileSystem(), core_options.GetManifestFormat(), + core_options.GetManifestCompression(), path_factory, + core_options.GetBucket(), memory_pool, core_options)); + return std::make_unique( + core_options.GetFileSystem(), std::move(index_manifest_file), + std::make_shared(path_factory), + core_options.DeletionVectorsBitmap64(), memory_pool); + } +}; + +/// @param loaded_schema The table's schema, when the caller already read it, or null when it has +/// to be read here. `TableScan::Create()` reads one to dispatch on its table type. +Result> NewDataTableScan( + const std::shared_ptr& context, const std::shared_ptr& loaded_schema); + +/// Maps a `ScanContext` onto `FormatTableScan`, which takes one partition filter and a limit and +/// nothing else. +/// +/// A format table has no buckets, so a bucket filter has nothing to act on. A predicate is +/// refused too, which is narrower than Java: `FormatReadBuilder.newScan()` hands the partition +/// half of one to the scan to prune directories with, which this does not do yet. +Result> NewFormatTableScan(const std::shared_ptr& table, + const std::shared_ptr& context) { + // Anything a `ScanContext` carries that a format table cannot honour is refused rather than + // silently dropped. + if (context->IsStreamingMode()) { + return Status::NotImplemented( + "a format table has no snapshots, so there is nothing for a streaming scan to follow"); + } + if (context->GetRealtimeContext() != nullptr) { + return Status::NotImplemented( + "a format table has no real-time store to union with what is on disk"); + } + if (context->GetGlobalIndexResult() != nullptr) { + return Status::NotImplemented("a format table carries no global index"); + } + std::map partition_filter; + std::shared_ptr filters = context->GetScanFilters(); + if (filters != nullptr) { + if (filters->GetPredicate() != nullptr) { + return Status::NotImplemented( + "a format table scan does not take a predicate: its files carry no statistics to " + "skip by, and pruning by the partition columns a predicate names is not " + "implemented yet; give the partition values as a partition filter instead"); + } + if (filters->GetBucketFilter()) { + return Status::NotImplemented("a format table has no buckets to filter by"); + } + const std::vector>& partition_filters = + filters->GetPartitionFilters(); + if (partition_filters.size() > 1) { + return Status::NotImplemented("a format table scan takes at most one partition filter"); + } + if (partition_filters.size() == 1) { + partition_filter = partition_filters.front(); + } + } + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan, + FormatTableScan::Create(table, partition_filter, context->GetLimit())); + return std::unique_ptr(std::move(scan)); +} + +} // namespace + +Result> TableScan::Create(std::unique_ptr context) { + if (context == nullptr) { + return Status::Invalid("scan context is null pointer"); + } + if (context->GetMemoryPool() == nullptr) { + return Status::Invalid("memory pool is null pointer"); + } + if (context->GetExecutor() == nullptr) { + return Status::Invalid("executor is null pointer"); + } + + std::shared_ptr shared_context = std::move(context); + // A table the caller already loaded says what it is, so nothing is read to find out. + if (shared_context->GetFormatTable() != nullptr) { + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr format_table, + FormatTable::Copy(shared_context->GetFormatTable(), shared_context->GetOptions())); + return NewFormatTableScan(format_table, shared_context); + } + // load schema + PAIMON_ASSIGN_OR_RAISE(CoreOptions tmp_options, + CoreOptions::FromMap(shared_context->GetOptions(), + shared_context->GetSpecificFileSystem(), {})); + PAIMON_ASSIGN_OR_RAISE(std::optional system_table_path, + SystemTableLoader::TryParsePath(shared_context->GetPath())); + if (system_table_path) { + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr system_table, + SystemTableLoader::LoadFromPath(tmp_options.GetFileSystem(), shared_context->GetPath(), + shared_context->GetOptions())); + return system_table->NewScan(shared_context); + } + // A format table is planned by listing directories, so it never reaches the manifest path + // below. One `TableScan` interface serves both, as Java Paimon serves both through one + // `ReadBuilder`. + std::shared_ptr latest_schema; + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr format_table, + FormatTableLoader::TryLoad(tmp_options.GetFileSystem(), shared_context->GetPath(), + BranchManager::NormalizeBranch(tmp_options.GetBranch()), + shared_context->GetOptions(), + shared_context->GetSpecificTableSchema(), + /*schema_manager=*/nullptr, &latest_schema)); + if (format_table != nullptr) { + return NewFormatTableScan(format_table, shared_context); + } + // With the schema the dispatch already read, so the managed path does not read it again. + return NewDataTableScan(shared_context, latest_schema); +} + +namespace { + +Status ValidateRealtimeScan(const TableSchema& table_schema, const CoreOptions& core_options, + const ScanContext& context, bool read_optimized) { + if (!context.GetRealtimeContext()) { + return Status::OK(); + } + if (!core_options.RealtimeEnabled()) { + return Status::Invalid("real-time scan requires realtime.enabled=true"); + } + if (core_options.GetBucket() <= 0) { + return Status::Invalid("real-time union read requires fixed bucket mode"); + } + if (core_options.DataEvolutionEnabled()) { + return Status::Invalid("real-time union read does not support data evolution"); + } + if (!table_schema.PrimaryKeys().empty()) { + if (read_optimized) { + return Status::NotImplemented( + "PK real-time union read does not support read-optimized scans"); + } + PAIMON_RETURN_NOT_OK( + PrimaryKeyTableUtils::ValidateRealtimeOptions(core_options, table_schema)); + } + if (context.IsStreamingMode()) { + return Status::Invalid("real-time union read currently supports batch scans only"); + } + if (context.GetLimit()) { + return Status::Invalid("real-time union read does not support scan limit pushdown"); + } + if (context.GetGlobalIndexResult()) { + return Status::Invalid("real-time union read does not support global index splits"); + } + StartupMode startup_mode = core_options.GetStartupMode(); + const bool specified_snapshot = + startup_mode == StartupMode::FromSnapshot() && core_options.GetScanSnapshotId().has_value(); + if (!(startup_mode == StartupMode::LatestFull() || startup_mode == StartupMode::Latest() || + specified_snapshot)) { + return Status::Invalid( + "real-time union read requires the latest snapshot or an explicit snapshot id"); + } + return Status::OK(); +} + +Result> NewDataTableScan( + const std::shared_ptr& context, + const std::shared_ptr& loaded_schema) { + PAIMON_ASSIGN_OR_RAISE( + CoreOptions tmp_options, + CoreOptions::FromMap(context->GetOptions(), context->GetSpecificFileSystem(), {})); + std::string branch = BranchManager::NormalizeBranch(tmp_options.GetBranch()); + std::shared_ptr table_schema; + const auto& specific_table_schema = context->GetSpecificTableSchema(); + if (loaded_schema != nullptr) { + table_schema = loaded_schema; + } else if (branch == BranchManager::DEFAULT_MAIN_BRANCH && specific_table_schema) { + PAIMON_ASSIGN_OR_RAISE(table_schema, + TableSchema::CreateFromJson(specific_table_schema.value())); + } else { + SchemaManager schema_manager(tmp_options.GetFileSystem(), context->GetPath(), branch); + PAIMON_ASSIGN_OR_RAISE(std::optional> latest_table_schema, + schema_manager.Latest()); + if (latest_table_schema == std::nullopt) { + return Status::Invalid("not found latest schema"); + } + table_schema = latest_table_schema.value(); + } + PAIMON_ASSIGN_OR_RAISE(bool read_optimized, + OptionsUtils::GetValueFromMap(context->GetOptions(), + kReadOptimizedScanOption, false)); + // merge options + auto options = table_schema->Options(); + for (const auto& [key, value] : context->GetOptions()) { + options[key] = value; + } + PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, + CoreOptions::FromMap(options, context->GetSpecificFileSystem(), {})); + core_options.WithCache(context->GetCache()); + + PAIMON_RETURN_NOT_OK(SchemaValidation::ValidateRealtime(*table_schema, core_options)); + + PAIMON_RETURN_NOT_OK( + ValidateRealtimeScan(*table_schema, core_options, *context, read_optimized)); + // validate options + if (core_options.GetBucket() == -1) { + if (!table_schema->PrimaryKeys().empty()) { + return Status::NotImplemented(fmt::format( + "do not support pk table bucket={} in scan process", core_options.GetBucket())); + } + } else if (core_options.GetBucket() < 1 && + !SchemaValidation::IsPostponeBucketTable(*table_schema, core_options.GetBucket())) { + return Status::Invalid( + fmt::format("do not support bucket={} in scan process", core_options.GetBucket())); + } + + // validate schema and scan filter + auto arrow_schema = DataField::ConvertDataFieldsToArrowSchema(table_schema->Fields()); + if (context->GetScanFilters() && context->GetScanFilters()->GetPredicate()) { + PAIMON_RETURN_NOT_OK(PredicateValidator::ValidatePredicateWithSchema( + *arrow_schema, context->GetScanFilters()->GetPredicate(), + /*validate_field_idx=*/false)); + } + PAIMON_ASSIGN_OR_RAISE(std::vector external_paths, + core_options.CreateExternalPaths()); + PAIMON_ASSIGN_OR_RAISE(std::optional global_index_external_path, + core_options.CreateGlobalIndexExternalPath()); + + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr path_factory, + FileStorePathFactory::Create( + context->GetPath(), arrow_schema, table_schema->PartitionKeys(), + core_options.GetPartitionDefaultName(), core_options.GetFileFormat()->Identifier(), + core_options.DataFilePrefix(), core_options.LegacyPartitionNameEnabled(), + external_paths, global_index_external_path, core_options.IndexFileInDataFileDir(), + context->GetMemoryPool())); + + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr file_store_scan, + TableScanImpl::CreateFileStoreScan( + path_factory, arrow_schema, table_schema, core_options, + context->GetExecutor(), context->GetMemoryPool(), context.get())); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr split_generator, + TableScanImpl::CreateSplitGenerator(table_schema, core_options, context.get())); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr index_file_handler, + TableScanImpl::CreateIndexFileHandler(core_options, path_factory, + context->GetMemoryPool())); + auto snapshot_reader = std::make_shared( + file_store_scan, path_factory, std::move(split_generator), std::move(index_file_handler)); + const bool pk_table = !table_schema->PrimaryKeys().empty(); + if (read_optimized && pk_table && context->IsStreamingMode()) { + return Status::NotImplemented( + "read-optimized system table does not support streaming scan for primary key table"); + } + if (context->IsStreamingMode()) { + return std::make_unique(core_options, snapshot_reader); + } + auto batch_scan = std::make_unique( + /*pk_table=*/pk_table, core_options, snapshot_reader, read_optimized, context->GetLimit(), + /*realtime_pk_scan=*/pk_table && context->GetRealtimeContext() != nullptr); + if (context->GetRealtimeContext()) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr realtime_context, + RealtimeContextImpl::Cast(context->GetRealtimeContext())); + return std::make_unique( + std::move(batch_scan), pk_table, realtime_context, path_factory, + snapshot_reader->GetSnapshotManager(), core_options.GetFileSystem(), + context->GetScanFilters(), core_options.GetRealtimeReadViewTtlMillis()); + } + if (core_options.DataEvolutionEnabled()) { + return std::make_unique( + context->GetPath(), snapshot_reader, std::move(batch_scan), table_schema, + context->GetGlobalIndexResult(), core_options, context->GetMemoryPool(), + context->GetExecutor()); + } + if (pk_table && !read_optimized && core_options.GlobalIndexEnabled()) { + PAIMON_ASSIGN_OR_RAISE(PrimaryKeyIndexDefinitions definitions, + PrimaryKeyIndexDefinitions::Create(*table_schema)); + if (!definitions.ScalarDefinitions().empty()) { + return PrimaryKeyIndexBatchScan::Create(snapshot_reader, std::move(batch_scan), + table_schema, path_factory, core_options, + context->GetMemoryPool()); + } + } + return batch_scan; +} + +} // namespace + +namespace { + +/// @param loaded_schema The schema of `branch`, when the caller already read it, or null when it +/// has to be read here: `TableRead::Create()` reads one to dispatch on the table type, and +/// reading it again would cost a second listing and read. +Result> CreateInternalReadContext( + const std::shared_ptr& context, const std::string& branch, + const std::shared_ptr& loaded_schema) { + std::map tmp_options = context->GetOptions(); + std::shared_ptr table_schema; + const auto& specific_table_schema = context->GetSpecificTableSchema(); + if (loaded_schema != nullptr) { + table_schema = loaded_schema; + } else if (branch == BranchManager::DEFAULT_MAIN_BRANCH && specific_table_schema) { + PAIMON_ASSIGN_OR_RAISE(table_schema, + TableSchema::CreateFromJson(specific_table_schema.value())); + } else { + PAIMON_ASSIGN_OR_RAISE(CoreOptions tmp_core_options, + CoreOptions::FromMap(tmp_options, context->GetSpecificFileSystem(), + context->GetFileSystemSchemeToIdentifierMap())); + SchemaManager schema_manager(tmp_core_options.GetFileSystem(), context->GetPath(), branch); + PAIMON_ASSIGN_OR_RAISE(std::optional> latest_schema, + schema_manager.Latest()); + if (!latest_schema) { + return Status::Invalid(fmt::format("schema file not found in path {}, branch {}", + context->GetPath(), branch)); + } + table_schema = latest_schema.value(); + } + assert(table_schema); + + // merge options + auto options = table_schema->Options(); + for (const auto& [key, value] : tmp_options) { + options[key] = value; + } + if (branch != BranchManager::DEFAULT_MAIN_BRANCH) { + options[Options::BRANCH] = branch; + } + return InternalReadContext::Create(context, table_schema, options); +} + +Result> CreateTableRead( + const std::shared_ptr& internal_context, + const std::shared_ptr& memory_pool, const std::shared_ptr& executor) { + const auto& core_options = internal_context->GetCoreOptions(); + const auto& table_schema = internal_context->GetTableSchema(); + if (internal_context->GetRealtimeContext() && !core_options.RealtimeEnabled()) { + return Status::Invalid("real-time read requires realtime.enabled=true"); + } + PAIMON_RETURN_NOT_OK(SchemaValidation::ValidateRealtime(*table_schema, core_options)); + auto arrow_schema = DataField::ConvertDataFieldsToArrowSchema(table_schema->Fields()); + PAIMON_ASSIGN_OR_RAISE(std::vector external_paths, + core_options.CreateExternalPaths()); + PAIMON_ASSIGN_OR_RAISE(std::optional global_index_external_path, + core_options.CreateGlobalIndexExternalPath()); + + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr path_factory, + FileStorePathFactory::Create( + internal_context->GetPath(), arrow_schema, table_schema->PartitionKeys(), + core_options.GetPartitionDefaultName(), core_options.GetFileFormat()->Identifier(), + core_options.DataFilePrefix(), core_options.LegacyPartitionNameEnabled(), + external_paths, global_index_external_path, core_options.IndexFileInDataFileDir(), + memory_pool)); + + if (internal_context->GetPrimaryKeys().empty()) { + return std::make_unique(path_factory, internal_context, memory_pool, + executor); + } + return KeyValueTableRead::Create(path_factory, internal_context, memory_pool, executor); +} + +Result> NewDataTableRead( + const std::shared_ptr& context, + const std::shared_ptr& loaded_schema) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr internal_context, + CreateInternalReadContext(context, context->GetBranch(), loaded_schema)); + + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr table_read, + CreateTableRead(internal_context, context->GetMemoryPool(), context->GetExecutor())); + + std::optional scan_fallback_branch = + internal_context->GetCoreOptions().GetScanFallbackBranch(); + if (!scan_fallback_branch || + StringUtils::IsNullOrWhitespaceOnly(scan_fallback_branch.value())) { + return std::move(table_read); + } + + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr fallback_context, + CreateInternalReadContext(context, /*branch=*/scan_fallback_branch.value(), + /*loaded_schema=*/nullptr)); + + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr fallback_table_read, + CreateTableRead(fallback_context, context->GetMemoryPool(), context->GetExecutor())); + const std::shared_ptr& arrow_pool = internal_context->GetArrowMemoryPool(); + return std::make_unique(std::move(table_read), + std::move(fallback_table_read), arrow_pool); +} + +/// Maps a `ReadContext` onto `FormatTableRead`, which reads everything it needs out of the context +/// itself: the columns, the predicate, the pool and executor, and what a file is opened with. It +/// refuses by name what a format table cannot honour. +Result> NewFormatTableRead(const std::shared_ptr& table, + const std::shared_ptr& context) { + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read, + FormatTableRead::Create(table, context)); + return std::unique_ptr(std::move(read)); +} + +} // namespace + +Result> TableRead::Create(std::unique_ptr ctx) { + std::shared_ptr context = std::move(ctx); + if (context == nullptr) { + return Status::Invalid("read context is null pointer"); + } + if (context->GetMemoryPool() == nullptr) { + return Status::Invalid("memory pool is null pointer"); + } + if (context->GetExecutor() == nullptr) { + return Status::Invalid("executor is null pointer"); + } + // A table the caller already loaded says what it is, so nothing is read to find out. + if (context->GetFormatTable() != nullptr) { + if (context->HasVectorOrFullTextSearch()) { + return Status::NotImplemented( + "VectorSearch or FullTextSearch is not supported for format tables"); + } + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr format_table, + FormatTable::Copy(context->GetFormatTable(), context->GetOptions())); + return NewFormatTableRead(format_table, context); + } + PAIMON_ASSIGN_OR_RAISE( + CoreOptions tmp_core_options, + CoreOptions::FromMap(context->GetOptions(), context->GetSpecificFileSystem(), + context->GetFileSystemSchemeToIdentifierMap())); + PAIMON_ASSIGN_OR_RAISE(std::optional system_table_path, + SystemTableLoader::TryParsePath(context->GetPath())); + if (system_table_path) { + if (context->HasVectorOrFullTextSearch()) { + return Status::NotImplemented( + "VectorSearch or FullTextSearch is not supported for system tables"); + } + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr system_table, + SystemTableLoader::LoadFromPath(tmp_core_options.GetFileSystem(), context->GetPath(), + context->GetOptions())); + return system_table->NewRead(context); + } + + // A format table has no manifest and a file another engine wrote carries no field ids, so + // which files there are and how a row is put back together is its own. How a file is opened + // is not: both paths go through `DataFileReaderFactory`. + std::shared_ptr latest_schema; + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr format_table, + FormatTableLoader::TryLoad(tmp_core_options.GetFileSystem(), context->GetPath(), + context->GetBranch(), context->GetOptions(), + context->GetSpecificTableSchema(), + /*schema_manager=*/nullptr, &latest_schema)); + if (format_table != nullptr) { + if (context->HasVectorOrFullTextSearch()) { + return Status::NotImplemented( + "VectorSearch or FullTextSearch is not supported for format tables"); + } + return NewFormatTableRead(format_table, context); + } + + // With the schema the dispatch already read, so the managed path does not read it again. + return NewDataTableRead(context, latest_schema); +} + +namespace { + +CommitScanner::ScanSupplier CreateAppendScanSupplier( + const std::shared_ptr& snapshot_manager, + const std::shared_ptr& schema_manager, + const std::shared_ptr& manifest_list, + const std::shared_ptr& manifest_file, + const std::shared_ptr& table_schema, + const std::shared_ptr& arrow_schema, const CoreOptions& options, + const std::shared_ptr& executor, const std::shared_ptr& pool) { + return [snapshot_manager, schema_manager, manifest_list, manifest_file, table_schema, + arrow_schema, options, executor, pool](const std::shared_ptr& scan_filter) + -> Result> { + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr scan, + AppendOnlyFileStoreScan::Create(snapshot_manager, schema_manager, manifest_list, + manifest_file, table_schema, arrow_schema, scan_filter, + options, executor, pool)); + return std::unique_ptr(std::move(scan)); + }; +} + +CommitScanner::ScanSupplier CreatePkScanSupplier( + const std::shared_ptr& snapshot_manager, + const std::shared_ptr& schema_manager, + const std::shared_ptr& manifest_list, + const std::shared_ptr& manifest_file, + const std::shared_ptr& table_schema, + const std::shared_ptr& arrow_schema, const CoreOptions& options, + const std::shared_ptr& executor, const std::shared_ptr& pool) { + return [snapshot_manager, schema_manager, manifest_list, manifest_file, table_schema, + arrow_schema, options, executor, pool](const std::shared_ptr& scan_filter) + -> Result> { + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr scan, + KeyValueFileStoreScan::Create(snapshot_manager, schema_manager, manifest_list, + manifest_file, table_schema, arrow_schema, scan_filter, + options, executor, pool)); + return std::unique_ptr(std::move(scan)); + }; +} + +bool IsCatalogCommit(const CommitContext& ctx) { + return AsVersionManaged(ctx.GetCatalog()) != nullptr; +} + +Result> NewSnapshotCommit( + const CommitContext& ctx, const std::optional& catalog_identifier, + const std::shared_ptr& fs, + const std::shared_ptr& snapshot_manager) { + if (IsCatalogCommit(ctx)) { + if (!catalog_identifier) { + return Status::Invalid("a catalog commit requires a table identifier"); + } + // Reuse the caller's UUID so table recreation cannot redirect this commit. + return std::shared_ptr(std::make_shared( + ctx.GetCatalog(), catalog_identifier.value(), ctx.GetTableId())); + } + if (ctx.UseRESTCatalogCommit()) { + return std::shared_ptr( + std::make_shared(ctx.GetTableId())); + } + // Catalogs without version management may still supply a table ID. + if (ctx.GetTableId() && ctx.GetCatalog() == nullptr) { + return Status::Invalid( + "a commit which publishes the snapshot by writing it to the table directory sends no " + "commit table request, so there is nowhere to put a table id; commit through a " + "catalog with WithCatalog() or ask for the request with UseRESTCatalogCommit()"); + } + return std::shared_ptr( + std::make_shared(fs, snapshot_manager)); +} + +/// Creates a format-table commit, rejecting non-default options that require snapshots. +Result> NewFormatTableCommit( + const std::shared_ptr& table, const CommitContext& ctx) { + if (!ctx.IgnoreEmptyCommit()) { + return Status::NotImplemented( + "a format table cannot record an empty commit: keeping one means writing a " + "snapshot that adds no files, and there are no snapshots here"); + } + if (ctx.UseRESTCatalogCommit()) { + return Status::NotImplemented( + "a format table commits by renaming files into place, not by sending a snapshot " + "to a rest catalog"); + } + if (ctx.GetTableId()) { + return Status::NotImplemented( + "a format table sends no commit table request, so there is nowhere to put a table id"); + } + if (ctx.GetCatalog() != nullptr) { + return Status::NotImplemented( + "a format table keeps no snapshot for a catalog to take, so it commits by renaming " + "files into place rather than through the catalog it was loaded from"); + } + if (ctx.AppendCommitCheckConflict()) { + return Status::NotImplemented( + "a format table has no manifests to check a concurrent commit against"); + } + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr format_commit, + FormatTableFileStoreCommit::Create(table)); + return std::unique_ptr(std::move(format_commit)); +} + +} // namespace + +Result> FileStoreCommit::Create( + std::unique_ptr ctx) { + if (ctx == nullptr) { + return Status::Invalid("commit context is null pointer"); + } + if (ctx->GetMemoryPool() == nullptr) { + return Status::Invalid("memory pool is null pointer"); + } + if (ctx->GetExecutor() == nullptr) { + return Status::Invalid("executor is null pointer"); + } + + // A table the caller already loaded says what it is, so nothing is read to find out. + if (ctx->GetFormatTable() != nullptr) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr given_table, + FormatTable::Copy(ctx->GetFormatTable(), ctx->GetOptions())); + return NewFormatTableCommit(given_table, *ctx); + } + + PAIMON_RETURN_NOT_OK(CheckVersionManagementImplemented(ctx->GetCatalog())); + // Use catalog credentials unless the caller supplied a file system. + std::shared_ptr specific_fs = ctx->GetSpecificFileSystem(); + if (specific_fs == nullptr && ctx->GetCatalog() != nullptr) { + if (!ctx->GetIdentifier()) { + return Status::Invalid("a catalog commit requires a table identifier"); + } + // Use the credentials of this table, which a catalog issuing per-table temporary + // ones only hands out through GetTableFileSystem. + PAIMON_ASSIGN_OR_RAISE(specific_fs, + ctx->GetCatalog()->GetTableFileSystem(ctx->GetIdentifier().value())); + } + PAIMON_ASSIGN_OR_RAISE(CoreOptions tmp_options, + CoreOptions::FromMap(ctx->GetOptions(), specific_fs)); + const std::string& root_path = ctx->GetRootPath(); + // The branch the context was built for, already validated by `CommitContextBuilder`. + PAIMON_ASSIGN_OR_RAISE(std::string branch, BranchManager::ResolveBranch( + ctx->GetIdentifier(), ctx->GetOptions(), + /*explicit_branch=*/std::nullopt, "commit")); + std::optional catalog_identifier; + if (ctx->GetIdentifier()) { + PAIMON_ASSIGN_OR_RAISE( + Identifier branch_identifier, + CatalogUtils::BranchIdentifier(ctx->GetIdentifier().value(), branch)); + catalog_identifier.emplace(std::move(branch_identifier)); + } + // Catalog-managed schemas may be absent from the table directory, but only on the main branch: + // a branch publishes its own under `branch/branch-`, which is where the load below and a + // read of the branch both read it, so a catalog answering for no branch schema does not fail + // a commit to one. + std::optional catalog_table_schema; + FileStoreCommitImpl::SchemaIdLoader schema_id_loader; + if (ctx->GetCatalog() != nullptr) { + if (!catalog_identifier) { + return Status::Invalid("a catalog commit requires a table identifier"); + } + if (BranchManager::IsMainBranch(branch)) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr loaded_schema, + ctx->GetCatalog()->LoadTableSchema(catalog_identifier.value())); + PAIMON_ASSIGN_OR_RAISE(std::string schema_json, loaded_schema->GetJsonSchema()); + catalog_table_schema = std::move(schema_json); + } + std::shared_ptr catalog = ctx->GetCatalog(); + Identifier identifier = catalog_identifier.value(); + schema_id_loader = [catalog, identifier]() -> Result { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, + catalog->LoadTableSchema(identifier)); + std::shared_ptr data_schema = std::dynamic_pointer_cast(schema); + if (data_schema == nullptr) { + return Status::Invalid( + "the catalog did not hand back a data table schema, which has the id a " + "snapshot records"); + } + return data_schema->Id(); + }; + } + // A format table commits by renaming files into place, so it never reaches the snapshot path + // below. The managed path here reads the commit's branch, so this reads the same one: the two + // must not dispatch on different schemas. + auto schema_manager = + std::make_shared(tmp_options.GetFileSystem(), root_path, branch); + std::shared_ptr latest_schema; + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr format_table, + FormatTableLoader::TryLoad( + tmp_options.GetFileSystem(), root_path, branch, ctx->GetOptions(), + catalog_table_schema, schema_manager.get(), &latest_schema)); + if (format_table != nullptr) { + return NewFormatTableCommit(format_table, *ctx); + } + if (latest_schema == nullptr) { + return Status::Invalid(fmt::format("not found latest schema in branch {}", branch)); + } + const std::shared_ptr& schema = latest_schema; + auto opts = schema->Options(); + for (const auto& [key, value] : ctx->GetOptions()) { + opts[key] = value; + } + // The metadata paths below are built for this branch, so what reads through these options, + // whose cache keys are scoped by it, has to read the same one. + opts[Options::BRANCH] = branch; + std::shared_ptr arrow_schema = + DataField::ConvertDataFieldsToArrowSchema(schema->Fields()); + PAIMON_ASSIGN_OR_RAISE(CoreOptions options, CoreOptions::FromMap(opts, specific_fs)); + PAIMON_RETURN_NOT_OK(SchemaValidation::ValidateRealtime(*schema, options)); + assert(options.GetFileSystem()); + assert(options.GetFileFormat()); + PAIMON_RETURN_NOT_OK(FileStoreCommitImpl::ValidateCommitOptions(options)); + PAIMON_ASSIGN_OR_RAISE(bool is_object_store, FileSystem::IsObjectStore(root_path)); + // Catalog commits do not require atomic rename on the object store. + if (is_object_store && !ctx->UseRESTCatalogCommit() && !IsCatalogCommit(*ctx) && + opts.find("enable-object-store-commit-in-inte-test") == opts.end()) { + return Status::NotImplemented( + "commit operation does not support object store file system for now"); + } + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr partition_computer, + BinaryRowPartitionComputer::Create(schema->PartitionKeys(), arrow_schema, + options.GetPartitionDefaultName(), + options.LegacyPartitionNameEnabled(), + ctx->GetMemoryPool())); + PAIMON_ASSIGN_OR_RAISE(std::vector external_paths, options.CreateExternalPaths()); + PAIMON_ASSIGN_OR_RAISE(std::optional global_index_external_path, + options.CreateGlobalIndexExternalPath()); + + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr path_factory, + FileStorePathFactory::Create( + root_path, arrow_schema, schema->PartitionKeys(), options.GetPartitionDefaultName(), + options.GetFileFormat()->Identifier(), options.DataFilePrefix(), + options.LegacyPartitionNameEnabled(), external_paths, global_index_external_path, + options.IndexFileInDataFileDir(), ctx->GetMemoryPool())); + + auto snapshot_manager = + std::make_shared(options.GetFileSystem(), root_path, branch); + if (IsCatalogCommit(*ctx)) { + std::shared_ptr catalog = ctx->GetCatalog(); + VersionManagedCatalog* versioned = AsVersionManaged(catalog); + Identifier identifier = catalog_identifier.value(); + // Capture the catalog to keep the versioned interface alive. + snapshot_manager->SetSnapshotLoader( + [catalog, versioned, identifier]() -> Result> { + return versioned->LoadSnapshot(identifier); + }); + } + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr manifest_list, + ManifestList::Create(options.GetFileSystem(), options.GetManifestFormat(), + options.GetManifestCompression(), path_factory, options.GetCache(), + ctx->GetMemoryPool())); + + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr partition_schema, + FieldMapping::GetPartitionSchema(arrow_schema, schema->PartitionKeys())); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr manifest_file, + ManifestFile::Create(options.GetFileSystem(), options.GetManifestFormat(), + options.GetManifestCompression(), path_factory, + options.GetManifestTargetFileSize(), ctx->GetMemoryPool(), options, + partition_schema)); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr index_manifest_file, + IndexManifestFile::Create(options.GetFileSystem(), options.GetManifestFormat(), + options.GetManifestCompression(), path_factory, + options.GetBucket(), ctx->GetMemoryPool(), options)); + + auto expire_snapshots = std::make_shared( + snapshot_manager, path_factory, manifest_list, manifest_file, index_manifest_file, + options.GetFileSystem(), options.GetExpireConfig(), options.RealtimeEnabled(), + ctx->GetExecutor()); + + CommitScanner::ScanSupplier scan_supplier; + if (schema->PrimaryKeys().empty()) { + scan_supplier = CreateAppendScanSupplier(snapshot_manager, schema_manager, manifest_list, + manifest_file, schema, arrow_schema, options, + ctx->GetExecutor(), ctx->GetMemoryPool()); + } else { + scan_supplier = CreatePkScanSupplier(snapshot_manager, schema_manager, manifest_list, + manifest_file, schema, arrow_schema, options, + ctx->GetExecutor(), ctx->GetMemoryPool()); + } + + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr snapshot_commit, + NewSnapshotCommit(*ctx, catalog_identifier, options.GetFileSystem(), snapshot_manager)); + + return std::make_unique( + ctx->GetMemoryPool(), ctx->GetExecutor(), arrow_schema, root_path, ctx->GetCommitUser(), + options, path_factory, std::move(partition_computer), snapshot_manager, snapshot_commit, + ctx->IgnoreEmptyCommit(), ctx->AppendCommitCheckConflict(), schema, + std::move(schema_id_loader), manifest_file, manifest_list, index_manifest_file, + expire_snapshots, schema_manager, std::move(scan_supplier)); +} + } // namespace paimon From b8e504b71926a8d6c7aa390d7e249c6d301588f2 Mon Sep 17 00:00:00 2001 From: "yonghao.fyh" Date: Mon, 19 Oct 2026 15:34:06 +0800 Subject: [PATCH 4/6] fix --- include/paimon/catalog/table.h | 10 -- .../core/catalog/file_system_catalog.cpp | 17 +++- src/paimon/core/table/table.cpp | 32 ------ src/paimon/core/table/table_test.cpp | 98 ++++++++++--------- 4 files changed, 68 insertions(+), 89 deletions(-) diff --git a/include/paimon/catalog/table.h b/include/paimon/catalog/table.h index 88a7e9b9e..e542c9aee 100644 --- a/include/paimon/catalog/table.h +++ b/include/paimon/catalog/table.h @@ -39,16 +39,6 @@ class TableRead; /// A table provides basic abstraction for table type. class PAIMON_EXPORT Table { public: - /// Creates a table by loading the latest schema from the specified table path. - /// - /// @param file_system File system used to access the table directory and schema files. - /// @param table_path Root path of the table in the file system. - /// @param identifier Logical table identifier selecting the schema branch and table name. - /// @return A table initialized with the latest available schema. - static Result> Create(const std::shared_ptr& file_system, - const std::string& table_path, - const Identifier& identifier); - /// @param file_system File system the table's data is accessed through. /// @param root_path Root path of the table in the file system. /// @param uuid Metastore UUID, or empty if unavailable. diff --git a/src/paimon/core/catalog/file_system_catalog.cpp b/src/paimon/core/catalog/file_system_catalog.cpp index d150f34bb..87c89de8c 100644 --- a/src/paimon/core/catalog/file_system_catalog.cpp +++ b/src/paimon/core/catalog/file_system_catalog.cpp @@ -345,9 +345,22 @@ Result> FileSystemCatalog::GetTable(const Identifier& ide PAIMON_ASSIGN_OR_RAISE(std::string table_path, GetTableLocation(data_identifier)); return std::make_shared
(fs_, table_path, schema, identifier); } + // The schema is read on the table's own branch: `root/branch/branch-` below the + // table path, which `main` resolves to itself. `GetBranchNameOrDefault` folds any + // spelling of `main`, so the branch read here agrees with `CurrentBranch()`. + PAIMON_ASSIGN_OR_RAISE(std::optional> latest_schema, + TableSchemaExists(identifier)); + if (!latest_schema) { + return Status::NotExist(fmt::format("{} not exist", identifier.ToString())); + } + // Only a managed table is a `Table`; describing another type as one would hand back + // snapshots and manifests it never had, so the failure belongs to the call that opens the + // table rather than at the first read. + PAIMON_RETURN_NOT_OK(CatalogUtils::CheckManagedTableType(identifier, *latest_schema, + "FileSystemCatalog::GetTable")); PAIMON_ASSIGN_OR_RAISE(std::string table_path, GetTableLocation(identifier)); - // `Table::Create` reads the same schema file and rejects a format table there. - return Table::Create(fs_, table_path, identifier); + return std::make_shared
(fs_, table_path, checked_pointer_cast(*latest_schema), + identifier); } Status FileSystemCatalog::DropDatabase(const std::string& name, bool ignore_if_not_exists, diff --git a/src/paimon/core/table/table.cpp b/src/paimon/core/table/table.cpp index 479f1037e..fb19038c8 100644 --- a/src/paimon/core/table/table.cpp +++ b/src/paimon/core/table/table.cpp @@ -37,7 +37,6 @@ #include "paimon/common/types/data_field.h" #include "paimon/common/utils/arrow/status_utils.h" #include "paimon/common/utils/binary_row_partition_computer.h" -#include "paimon/common/utils/checked_cast.h" #include "paimon/common/utils/fields_comparator.h" #include "paimon/common/utils/options_utils.h" #include "paimon/common/utils/string_utils.h" @@ -215,37 +214,6 @@ Table::Table(const std::shared_ptr& file_system, const std::string& identifier_(identifier), uuid_(uuid) {} -Result> Table::Create(const std::shared_ptr& file_system, - const std::string& table_path, - const Identifier& identifier) { - PAIMON_ASSIGN_OR_RAISE(bool exist, file_system->Exists(table_path)); - if (!exist) { - return Status::NotExist(fmt::format("{} not exist", identifier.ToString())); - } - - // The schema is read on the table's own branch: `root/branch/branch-` below - // the table path, which `main` resolves to itself. `GetBranchNameOrDefault` folds - // any spelling of `main`, so the branch read here agrees with `CurrentBranch()`. - PAIMON_RETURN_NOT_OK(CatalogUtils::CheckValidTableName(identifier)); - PAIMON_ASSIGN_OR_RAISE(std::string branch, identifier.GetBranchNameOrDefault()); - SchemaManager schema_manager(file_system, table_path, branch); - PAIMON_ASSIGN_OR_RAISE(std::optional> latest_schema, - schema_manager.Latest()); - if (!latest_schema) { - return Status::NotExist( - fmt::format("load table schema for {} failed", identifier.ToString())); - } - - // Only a managed table is a `Table`; describing another type as one would hand back snapshots - // and manifests it never had. Checked here because this is the one place every `Table` built - // from a table directory passes through, catalogs included. - PAIMON_RETURN_NOT_OK( - CatalogUtils::CheckManagedTableType(identifier, *latest_schema, "Table::Create")); - - auto schema = checked_pointer_cast(*latest_schema); - return std::make_shared
(file_system, table_path, schema, identifier, /*uuid=*/""); -} - Result> Table::NewScan(std::unique_ptr context) const { return TableScan::Create(std::move(context)); } diff --git a/src/paimon/core/table/table_test.cpp b/src/paimon/core/table/table_test.cpp index da0cae11d..4792a8461 100644 --- a/src/paimon/core/table/table_test.cpp +++ b/src/paimon/core/table/table_test.cpp @@ -27,6 +27,8 @@ #include "arrow/api.h" #include "gtest/gtest.h" #include "paimon/commit_context.h" +#include "paimon/common/utils/checked_cast.h" +#include "paimon/core/catalog/file_system_catalog.h" #include "paimon/core/schema/schema_manager.h" #include "paimon/core/schema/table_schema.h" #include "paimon/file_store_commit.h" @@ -44,27 +46,34 @@ namespace paimon::test { namespace { -// Creates a managed table directory and opens it as a `Table` under `identifier`. The -// schema goes to `branch`'s directory, which a branch identifier reads it back from. +// Creates a managed table directory and a `Table` over it, holding the schema written to +// `branch`'s directory under `identifier`. Result> CreateManagedTable(const std::shared_ptr& file_system, const std::string& path, const Identifier& identifier, const std::string& branch = "main") { SchemaManager schema_manager(file_system, path, branch); auto schema = arrow::schema({arrow::field("id", arrow::int32(), /*nullable=*/false)}); - PAIMON_ASSIGN_OR_RAISE([[maybe_unused]] std::unique_ptr created, + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr created, schema_manager.CreateTable(schema, /*partition_keys=*/{}, /*primary_keys=*/{}, /*options=*/{})); - return Table::Create(file_system, path, identifier); + return std::make_shared
( + file_system, path, + checked_pointer_cast(std::shared_ptr(std::move(created))), identifier); } } // namespace -TEST(TableTest, TestCreateWithUnknownDatabase) { +TEST(TableTest, TestGetTableWithUnknownDatabase) { auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); + auto fs = dir->GetFileSystem(); + // The database the identifier names lands in its own directory below the warehouse, so the + // table the catalog reads back is the one that identifier selects. + const std::string table_dir = dir->Str() + "/unknown.db/tbl1"; + ASSERT_OK(fs->Mkdirs(table_dir)); - SchemaManager schema_manager(dir->GetFileSystem(), dir->Str()); + SchemaManager schema_manager(fs, table_dir); auto schema = arrow::schema({arrow::field("id", arrow::int32(), /*nullable=*/false), arrow::field("name", arrow::utf8()), arrow::field("value", arrow::int64())}); @@ -78,9 +87,9 @@ TEST(TableTest, TestCreateWithUnknownDatabase) { ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr created_schema, schema_manager.CreateTable(schema, partition_keys, primary_keys, options)); + FileSystemCatalog catalog(fs, dir->Str(), /*catalog_options=*/{}); ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, - Table::Create(dir->GetFileSystem(), dir->Str(), - Identifier(Identifier::kUnknownDatabase, "tbl1"))); + catalog.GetTable(Identifier(Identifier::kUnknownDatabase, "tbl1"))); EXPECT_EQ(table->Name(), "tbl1"); EXPECT_EQ(table->FullName(), "tbl1"); @@ -95,34 +104,35 @@ TEST(TableTest, TestCreateWithUnknownDatabase) { EXPECT_EQ(data_schema->PrimaryKeys(), primary_keys); } -TEST(TableTest, TestCreateWithBranch) { +TEST(TableTest, TestGetTableWithBranch) { auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); auto fs = dir->GetFileSystem(); - SchemaManager schema_manager(fs, dir->Str()); + const std::string table_dir = dir->Str() + "/db1.db/tbl1"; + ASSERT_OK(fs->Mkdirs(table_dir)); + SchemaManager schema_manager(fs, table_dir); ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr main_schema, schema_manager.CreateTable( arrow::schema({arrow::field("main_c0", arrow::int32())}), {}, {}, {})); - SchemaManager branch_schema_manager(fs, dir->Str(), "b1"); + SchemaManager branch_schema_manager(fs, table_dir, "b1"); ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr branch_schema, branch_schema_manager.CreateTable( arrow::schema({arrow::field("b1_c0", arrow::int32())}), {}, {}, {})); + FileSystemCatalog catalog(fs, dir->Str(), /*catalog_options=*/{}); ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, - Table::Create(fs, dir->Str(), Identifier("db1", "tbl1$branch_b1"))); + catalog.GetTable(Identifier("db1", "tbl1$branch_b1"))); ASSERT_EQ("tbl1$branch_b1", table->Name()); ASSERT_EQ((std::vector{"b1_c0"}), table->LatestSchema()->FieldNames()); for (const auto& main_identifier : {Identifier("db1", "tbl1$branch_main"), Identifier("db1", "tbl1$branch_MAIN")}) { - ASSERT_OK_AND_ASSIGN(table, Table::Create(fs, dir->Str(), main_identifier)); + ASSERT_OK_AND_ASSIGN(table, catalog.GetTable(main_identifier)); ASSERT_EQ((std::vector{"main_c0"}), table->LatestSchema()->FieldNames()); } - Status missing_branch = - Table::Create(fs, dir->Str(), Identifier("db1", "tbl1$branch_missing")).status(); + Status missing_branch = catalog.GetTable(Identifier("db1", "tbl1$branch_missing")).status(); ASSERT_TRUE(missing_branch.IsNotExist()) << missing_branch.ToString(); - ASSERT_NOK_WITH_MSG(Table::Create(fs, dir->Str(), Identifier("db1", "tbl1$branch_../outside")), - "branch"); + ASSERT_NOK_WITH_MSG(catalog.GetTable(Identifier("db1", "tbl1$branch_../outside")), "branch"); } TEST(TableTest, TestCreateHoldsTheFileSystem) { @@ -143,27 +153,21 @@ TEST(TableTest, TestMainBranchAliasResolvesToTheMainBranch) { ASSERT_TRUE(dir); auto fs = dir->GetFileSystem(); SchemaManager schema_manager(fs, dir->Str()); - ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr schema, + ASSERT_OK_AND_ASSIGN(std::unique_ptr created, schema_manager.CreateTable( arrow::schema({arrow::field("c0", arrow::int32())}), {}, {}, {})); + auto schema = checked_pointer_cast(std::shared_ptr(std::move(created))); for (const auto& main_identifier : {Identifier("db", "tbl"), Identifier("db", "tbl$branch_main"), Identifier("db", "tbl$branch_MAIN")}) { SCOPED_TRACE(main_identifier.ToString()); - ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, - Table::Create(fs, dir->Str(), main_identifier)); - ASSERT_OK_AND_ASSIGN(std::string branch, table->CurrentBranch()); + // A catalog that loaded the schema itself calls the constructor directly, which a + // rest catalog is shaped like. The alias resolves to the main branch as well. + Table table(fs, dir->Str(), schema, main_identifier); + ASSERT_OK_AND_ASSIGN(std::string branch, table.CurrentBranch()); EXPECT_EQ("main", branch); } - - // A catalog that loaded the schema itself calls the constructor directly, which a - // rest catalog is shaped like. The alias resolves to the main branch as well. - ASSERT_OK_AND_ASSIGN(std::shared_ptr
plain, - Table::Create(fs, dir->Str(), Identifier("db", "tbl"))); - Table alias_table(fs, dir->Str(), plain->LatestSchema(), Identifier("db", "tbl$branch_MAIN")); - ASSERT_OK_AND_ASSIGN(std::string alias_branch, alias_table.CurrentBranch()); - EXPECT_EQ("main", alias_branch); } TEST(TableTest, TestCurrentBranch) { @@ -207,40 +211,44 @@ TEST(TableTest, TestCatalogOpenedTableServesThePerTableFileSystem) { EXPECT_EQ(opened->GetRootPath(), ""); } -TEST(TableTest, TestCreateRejectsATableTypeThatIsNotAManagedTable) { +TEST(TableTest, TestGetTableRejectsATableTypeThatIsNotAManagedTable) { // A `Table` describes a managed paimon table. Handing back another type as one would promise // snapshots and manifests it never had, so the failure belongs at the call that opens the - // table rather than at the first read. This is the one place every `Table` built from a table - // directory passes through, catalogs included. + // table rather than at the first read. auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); + auto fs = dir->GetFileSystem(); auto schema = arrow::schema({arrow::field("id", arrow::int32(), /*nullable=*/false), arrow::field("dt", arrow::utf8())}); - SchemaManager format_schema_manager(dir->GetFileSystem(), dir->Str() + "/format"); + FileSystemCatalog catalog(fs, dir->Str(), /*catalog_options=*/{}); + const std::string format_dir = dir->Str() + "/db.db/format"; + ASSERT_OK(fs->Mkdirs(format_dir)); + SchemaManager format_schema_manager(fs, format_dir); ASSERT_OK_AND_ASSIGN( [[maybe_unused]] std::unique_ptr format_schema, format_schema_manager.CreateTable(schema, /*partition_keys=*/{"dt"}, /*primary_keys=*/{}, {{"type", "format-table"}, {"file.format", "parquet"}})); - ASSERT_NOK_WITH_MSG( - Table::Create(dir->GetFileSystem(), dir->Str() + "/format", Identifier("db", "fmt")), - "Cannot open format table"); + ASSERT_NOK_WITH_MSG(catalog.GetTable(Identifier("db", "format")), "Cannot open format table"); // A managed table with no `type` option at all is what a `Table` is for, and still opens. - SchemaManager managed_schema_manager(dir->GetFileSystem(), dir->Str() + "/managed"); + const std::string managed_dir = dir->Str() + "/db.db/managed"; + ASSERT_OK(fs->Mkdirs(managed_dir)); + SchemaManager managed_schema_manager(fs, managed_dir); ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr managed_schema, managed_schema_manager.CreateTable(schema, /*partition_keys=*/{"dt"}, /*primary_keys=*/{}, {})); - ASSERT_OK( - Table::Create(dir->GetFileSystem(), dir->Str() + "/managed", Identifier("db", "managed"))); + ASSERT_OK(catalog.GetTable(Identifier("db", "managed"))); } -TEST(TableTest, TestCreateFailedWithNonExistSchema) { +TEST(TableTest, TestGetTableFailedWithNonExistSchema) { auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); auto fs = dir->GetFileSystem(); - SchemaManager schema_manager(fs, dir->Str()); + const std::string table_dir = dir->Str() + "/unknown.db/tbl1"; + ASSERT_OK(fs->Mkdirs(table_dir)); + SchemaManager schema_manager(fs, table_dir); auto schema = arrow::schema({arrow::field("id", arrow::int32(), /*nullable=*/false), arrow::field("name", arrow::utf8()), arrow::field("value", arrow::int64())}); @@ -257,10 +265,10 @@ TEST(TableTest, TestCreateFailedWithNonExistSchema) { // remove schema std::string schema_path = schema_manager.ToSchemaPath(0); ASSERT_OK(fs->Delete(schema_path, /*recursive=*/false)); - // check create table failed - ASSERT_NOK_WITH_MSG( - Table::Create(fs, dir->Str(), Identifier(Identifier::kUnknownDatabase, "tbl1")), - "load table schema for Identifier{database='unknown', object='tbl1'} failed"); + // check get table failed + FileSystemCatalog catalog(fs, dir->Str(), /*catalog_options=*/{}); + ASSERT_NOK_WITH_MSG(catalog.GetTable(Identifier(Identifier::kUnknownDatabase, "tbl1")), + "not exist"); } TEST(TableTest, TestNewScanCreatesATableScan) { From 1329110defeeef9845926d4d20e21797163dc381 Mon Sep 17 00:00:00 2001 From: "yonghao.fyh" Date: Mon, 19 Oct 2026 16:08:53 +0800 Subject: [PATCH 5/6] fix --- src/paimon/core/catalog/identifier_test.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/paimon/core/catalog/identifier_test.cpp b/src/paimon/core/catalog/identifier_test.cpp index 4b3c19f82..c4d5823d2 100644 --- a/src/paimon/core/catalog/identifier_test.cpp +++ b/src/paimon/core/catalog/identifier_test.cpp @@ -112,7 +112,7 @@ TEST(IdentifierTest, MainBranchAliasFoldsInGetBranchNameOrDefault) { // `GetBranchName` keeps the spelling the object name was written with, while // `GetBranchNameOrDefault` folds any spelling of the main branch to `main`: the value // a manager is bound to, which names no directory of its own. - for (const std::string& alias : {"MAIN", "Main", "mAiN"}) { + for (const std::string alias : {"MAIN", "Main", "mAiN"}) { Identifier id("db", "tbl$branch_" + alias); ASSERT_OK_AND_ASSIGN(std::optional branch_name, id.GetBranchName()); ASSERT_TRUE(branch_name); From e9e7a6606074b79bd4ab944d52308ec7dc71eaf0 Mon Sep 17 00:00:00 2001 From: "yonghao.fyh" Date: Mon, 19 Oct 2026 16:36:25 +0800 Subject: [PATCH 6/6] fix --- include/paimon/read_context.h | 37 +++++++++++++++------------------ include/paimon/scan_context.h | 4 ++-- src/paimon/core/table/table.cpp | 13 ++++++------ 3 files changed, 25 insertions(+), 29 deletions(-) diff --git a/include/paimon/read_context.h b/include/paimon/read_context.h index 716110c5e..28305c94c 100644 --- a/include/paimon/read_context.h +++ b/include/paimon/read_context.h @@ -52,26 +52,23 @@ struct VectorSearch; /// @see ReadContextBuilder class PAIMON_EXPORT ReadContext { public: - ReadContext(const std::string& path, const std::string& branch, - const std::vector& read_field_names, - const std::vector& read_field_ids, - const std::shared_ptr& predicate, - const std::shared_ptr& vector_search, - const std::shared_ptr& full_text_search, - bool enable_predicate_filter, bool enable_prefetch, bool enable_late_materializing, - uint32_t prefetch_batch_count, uint32_t prefetch_max_parallel_num, - bool enable_multi_thread_row_to_batch, uint32_t row_to_batch_thread_number, - const std::optional& table_schema, - const std::shared_ptr& memory_pool, - const std::shared_ptr& executor, - const std::shared_ptr& specific_file_system, - const std::map& fs_scheme_to_identifier_map, - const std::shared_ptr& realtime_context, - const std::map& options, bool read_ahead_cache_enabled, - const CacheConfig& cache_config, const std::shared_ptr& cache, - const std::shared_ptr& format_table, WarmupLevel warmup_level, - const std::shared_ptr& catalog = nullptr, - const std::optional& identifier = std::nullopt); + ReadContext( + const std::string& path, const std::string& branch, + const std::vector& read_field_names, + const std::vector& read_field_ids, const std::shared_ptr& predicate, + const std::shared_ptr& vector_search, + const std::shared_ptr& full_text_search, bool enable_predicate_filter, + bool enable_prefetch, bool enable_late_materializing, uint32_t prefetch_batch_count, + uint32_t prefetch_max_parallel_num, bool enable_multi_thread_row_to_batch, + uint32_t row_to_batch_thread_number, const std::optional& table_schema, + const std::shared_ptr& memory_pool, const std::shared_ptr& executor, + const std::shared_ptr& specific_file_system, + const std::map& fs_scheme_to_identifier_map, + const std::shared_ptr& realtime_context, + const std::map& options, bool read_ahead_cache_enabled, + const CacheConfig& cache_config, const std::shared_ptr& cache, + const std::shared_ptr& format_table, WarmupLevel warmup_level, + const std::shared_ptr& catalog, const std::optional& identifier); ~ReadContext(); diff --git a/include/paimon/scan_context.h b/include/paimon/scan_context.h index fac2a1a35..6fbb006f0 100644 --- a/include/paimon/scan_context.h +++ b/include/paimon/scan_context.h @@ -59,8 +59,8 @@ class PAIMON_EXPORT ScanContext { const std::map& options, const std::shared_ptr& cache, const std::shared_ptr& format_table, - const std::shared_ptr& catalog = nullptr, - const std::optional& identifier = std::nullopt); + const std::shared_ptr& catalog, + const std::optional& identifier); ~ScanContext(); diff --git a/src/paimon/core/table/table.cpp b/src/paimon/core/table/table.cpp index fb19038c8..bc7085521 100644 --- a/src/paimon/core/table/table.cpp +++ b/src/paimon/core/table/table.cpp @@ -7,14 +7,13 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ #include "paimon/catalog/table.h"