From 7c6fec5922004d76d58b1e829f0bbfc15aa9bd42 Mon Sep 17 00:00:00 2001 From: "yonghao.fyh" Date: Tue, 20 Oct 2026 14:58:24 +0800 Subject: [PATCH 01/17] refactor table --- docs/source/user_guide/format_table.rst | 28 +- include/paimon/catalog/catalog.h | 33 +- include/paimon/catalog/table.h | 47 ++- include/paimon/commit_context.h | 2 +- include/paimon/read_context.h | 2 +- include/paimon/scan_context.h | 2 +- include/paimon/table/format/format_table.h | 74 ++-- include/paimon/write_context.h | 2 +- src/paimon/CMakeLists.txt | 1 - src/paimon/core/catalog/catalog.cpp | 17 - src/paimon/core/catalog/catalog_utils.cpp | 20 - src/paimon/core/catalog/catalog_utils.h | 25 +- .../core/catalog/file_system_catalog.cpp | 30 +- src/paimon/core/catalog/file_system_catalog.h | 7 - .../core/catalog/file_system_catalog_test.cpp | 21 +- src/paimon/core/table/file_store_table.h | 65 +++ src/paimon/core/table/format/format_table.cpp | 195 ++++++++- .../core/table/format/format_table_loader.cpp | 87 ---- .../core/table/format/format_table_loader.h | 71 ---- .../core/table/format/format_table_test.cpp | 155 +++++++ src/paimon/core/table/table.cpp | 389 +++++++----------- src/paimon/core/table/table_test.cpp | 18 +- src/paimon/rest/rest_catalog.cpp | 37 +- src/paimon/rest/rest_catalog.h | 6 - src/paimon/rest/rest_catalog_test.cpp | 45 +- src/paimon/testing/mock/mock_catalog.h | 3 +- 26 files changed, 755 insertions(+), 627 deletions(-) create mode 100644 src/paimon/core/table/file_store_table.h delete mode 100644 src/paimon/core/table/format/format_table_loader.cpp delete mode 100644 src/paimon/core/table/format/format_table_loader.h diff --git a/docs/source/user_guide/format_table.rst b/docs/source/user_guide/format_table.rst index 492b5f0d0..4be21a1dd 100644 --- a/docs/source/user_guide/format_table.rst +++ b/docs/source/user_guide/format_table.rst @@ -40,8 +40,9 @@ format table is an insert. Reading and writing ------------------- -A format table is not served through :cpp:func:`Catalog::GetTable`, which describes a managed -table; use :cpp:func:`Catalog::GetFormatTable` instead. +A format table is served through the same :cpp:func:`Catalog::GetTable` as every other table: +its ``type`` option says what it is, and the call hands back the ``FormatTable`` it describes, +through the ``Table`` interface the two kinds of table share. Reading and writing go through the entry points every other table uses: ``TableScan::Create``, ``TableRead::Create``, ``FileStoreWrite::Create`` and ``FileStoreCommit::Create``, each from its @@ -58,8 +59,11 @@ a constructor taking one - which is what Java does through ``FormatTable.newRead .. code-block:: cpp - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table, - catalog->GetFormatTable(paimon::Identifier("db", "tbl"))); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr loaded, + catalog->GetTable(paimon::Identifier("db", "tbl"))); + // The table's `type` option is `format-table`, so this is a `FormatTable`. + std::shared_ptr table = + std::dynamic_pointer_cast(loaded); paimon::ScanContextBuilder scan_builder(table); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan_context, scan_builder.Finish()); @@ -71,6 +75,12 @@ from one refuses ``SetTableSchema()``, ``WithFileSystem()``, ``WithFileSystemSchemeToIdentifierMap()``, ``WithCatalog()`` and a branch. Options given at the call still win over the ones the schema stored, as they do everywhere else. +``FormatTable`` is a derived class of ``Table``, so the table serves the same calls itself: a +caller holding one through a ``std::shared_ptr`` calls ``NewScan()``, +``NewRead()``, ``NewWrite()`` or ``NewCommit()`` on it, and the operators come back for the table +it is, whether or not the context carries one. This is the C++ side of what Java serves through +one ``Table.newReadBuilder()`` for both kinds of table. + A batch comes back in the table's column order, or the projection's when the read names one, behind the ``_VALUE_KIND`` field every ``BatchReader`` carries. Which columns a *file* is asked for and in what order is its own business: they are matched to the table's by name, as Java matches @@ -89,12 +99,10 @@ outside the projection is never read, which is how Java scopes the same checks. named here only to describe what happens. A caller never needs them: a plan comes back as ``Plan``, a split as ``Split``, and a commit message as ``CommitMessage``. -A catalog that can load a format table itself overrides ``Catalog::LoadFormatTable()``, the -protected hook :cpp:func:`Catalog::GetFormatTable` calls. The file system catalog uses it to say -its metadata lives under the table location, and the REST catalog to take the location and the -schema from one response instead of two that could disagree. A catalog that does not override it -still serves format tables, by reading the location and the schema through the virtuals every -catalog has. +Each catalog serves a format table from its own ``GetTable``. The file system catalog reads the +schema under the table location, so it says the location carries the table's own metadata and a +scan leaves the ``schema`` and ``branch`` directories below it alone; the REST catalog takes the +location and the schema from one response, so the two cannot disagree. ``TableScan::ListPartitions()`` lists the partitions a scan can see. A format table answers it by listing directories; every other table type returns ``NotImplemented`` for now. diff --git a/include/paimon/catalog/catalog.h b/include/paimon/catalog/catalog.h index 69eb6187c..d501bc7e3 100644 --- a/include/paimon/catalog/catalog.h +++ b/include/paimon/catalog/catalog.h @@ -40,7 +40,6 @@ namespace paimon { using Instant = std::variant; class Database; -class FormatTable; class Table; class View; class Schema; @@ -148,6 +147,11 @@ class PAIMON_EXPORT Catalog { /// Gets a table. /// + /// The table's `type` option decides what is handed back: a `format-table` comes back as + /// the `FormatTable` it is - a directory of data files, with no snapshot and no manifest - + /// and every other implemented type as the managed table it is. Both go through this one + /// `Table` interface: `NewScan()`, `NewRead()`, `NewWrite()` and `NewCommit()` serve either. + /// /// @param identifier Identifier of the table to get. /// @return A result containing the table, or an error status. virtual Result> GetTable(const Identifier& identifier) const = 0; @@ -246,33 +250,6 @@ class PAIMON_EXPORT Catalog { /// @note Implementations returning true must also implement the internal /// `VersionManagedCatalog`. virtual bool SupportsVersionManagement() const; - - /// Gets a format table: a directory of data files laid out like a standard Hive table. - /// - /// A format table carries no snapshots and no manifests, so it is loaded through its own - /// method rather than `GetTable()`. Reading and writing it go through the same `TableScan`, - /// `TableRead`, `FileStoreWrite` and `FileStoreCommit` entry points every other table uses: - /// hand the table back to `ScanContextBuilder`, `ReadContextBuilder`, `WriteContextBuilder` or - /// `CommitContextBuilder`, each of which takes one. - /// - /// @param identifier Identifier of the table to get. - /// @return A result containing the format table, or an error status if the table does not - /// exist or its `type` option is not `format-table`. - Result> GetFormatTable(const Identifier& identifier) const; - - protected: - /// Loads `identifier` as a format table, which is what `GetFormatTable()` hands back. - /// - /// The default reads the location and the schema through the virtuals above - two requests - /// that can disagree - and treats every directory below the location as table content. An - /// override answers both questions directly: whether location and schema come back in one - /// round trip, and whether this catalog keeps the table's metadata under the table path. - /// - /// @param identifier Identifier of the table to load. - /// @return A result containing the format table, or an error status if the table does not - /// exist or its `type` option is not `format-table`. - virtual Result> LoadFormatTable( - const Identifier& identifier) const; }; } // namespace paimon diff --git a/include/paimon/catalog/table.h b/include/paimon/catalog/table.h index e542c9aee..d18bd07b8 100644 --- a/include/paimon/catalog/table.h +++ b/include/paimon/catalog/table.h @@ -33,20 +33,25 @@ namespace paimon { -class TableScan; +class CommitContext; +class FileStoreCommit; +class FileStoreWrite; +class ReadContext; +class ScanContext; class TableRead; +class TableScan; +class WriteContext; /// A table provides basic abstraction for table type. +/// +/// The state a table of every kind carries - file system, root path, schema and identifier - lives +/// here, and each kind of table fills the operations in: the operators below create this table's +/// scan, read, write or commit, from a context the caller finishes through its builder. What the +/// context asks for that this table cannot honour is refused by name by the implementation rather +/// than silently dropped. class PAIMON_EXPORT Table { public: - /// @param file_system File system the table's data is accessed through. - /// @param root_path Root path of the table in the file system. - /// @param uuid Metastore UUID, or empty if unavailable. - Table(const std::shared_ptr& file_system, const std::string& root_path, - const std::shared_ptr& schema, const Identifier& identifier, - const std::string& uuid = ""); - - ~Table() = default; + virtual ~Table() = default; /// Returns the file system the table was opened with, which serves the table's /// files. A catalog that hands out per-table file systems opens a table with the @@ -99,33 +104,45 @@ class PAIMON_EXPORT Table { /// that `TableScan::Create()` also builds through. The scan is derived from the context. /// @param context The finished scan context. /// @return The scan, or the error building it reports. - Result> NewScan(std::unique_ptr context) const; + virtual Result> NewScan( + std::unique_ptr context) const = 0; /// Creates a read of this table from a finished read context, the table-first entry point /// that `TableRead::Create()` also builds through. The read is derived from the context. /// @param context The finished read context. /// @return The read, or the error building it reports. - Result> NewRead(std::unique_ptr context) const; + virtual Result> NewRead( + std::unique_ptr context) const = 0; /// Creates a write of this table from a finished write context, the table-first entry point /// that `FileStoreWrite::Create()` also builds through. The write is derived from the context. /// @param context The finished write context. /// @return The write, or the error building it reports. - Result> NewWrite(std::unique_ptr context) const; + virtual Result> NewWrite( + std::unique_ptr context) const = 0; /// Creates a commit of this table from a finished commit context, the table-first entry point /// that `FileStoreCommit::Create()` also builds through. The commit is derived from the /// context. /// @param context The finished commit context. /// @return The commit, or the error building it reports. - Result> NewCommit( - std::unique_ptr context) const; + virtual Result> NewCommit( + std::unique_ptr context) const = 0; + + protected: + /// @param file_system File system the table's data is accessed through. + /// @param root_path Root path of the table in the file system. + /// @param uuid Metastore UUID, or empty if unavailable. + Table(const std::shared_ptr& file_system, const std::string& root_path, + const std::shared_ptr& schema, const Identifier& identifier, + const std::string& uuid = ""); - private: std::shared_ptr file_system_; std::string root_path_; std::shared_ptr schema_; Identifier identifier_; + + private: std::string uuid_; }; diff --git a/include/paimon/commit_context.h b/include/paimon/commit_context.h index ffd40ac07..1ec784f76 100644 --- a/include/paimon/commit_context.h +++ b/include/paimon/commit_context.h @@ -151,7 +151,7 @@ class PAIMON_EXPORT CommitContextBuilder { /// /// There is no commit user: a format table keeps no snapshot to record one in. /// - /// @param table The format table to commit to, as `Catalog::GetFormatTable()` hands it back. + /// @param table The format table to commit to, as `Catalog::GetTable()` hands it back. explicit CommitContextBuilder(const std::shared_ptr& table); ~CommitContextBuilder(); diff --git a/include/paimon/read_context.h b/include/paimon/read_context.h index e62386c65..b9c534065 100644 --- a/include/paimon/read_context.h +++ b/include/paimon/read_context.h @@ -246,7 +246,7 @@ class PAIMON_EXPORT ReadContextBuilder { /// `SetTableSchema()`, `WithFileSystem()`, `WithFileSystemSchemeToIdentifierMap()` and a /// branch are refused here rather than ignored. /// - /// @param table The format table to read, as `Catalog::GetFormatTable()` hands it back. + /// @param table The format table to read, as `Catalog::GetTable()` hands it back. explicit ReadContextBuilder(const std::shared_ptr& table); ~ReadContextBuilder(); diff --git a/include/paimon/scan_context.h b/include/paimon/scan_context.h index 6fbb006f0..5be7c9c4e 100644 --- a/include/paimon/scan_context.h +++ b/include/paimon/scan_context.h @@ -183,7 +183,7 @@ class PAIMON_EXPORT ScanContextBuilder { /// table a REST catalog serves. The table carries what such a location does not say, so /// `SetTableSchema()` and `WithFileSystem()` are refused here rather than ignored. /// - /// @param table The format table to scan, as `Catalog::GetFormatTable()` hands it back. + /// @param table The format table to scan, as `Catalog::GetTable()` hands it back. explicit ScanContextBuilder(const std::shared_ptr& table); ~ScanContextBuilder(); diff --git a/include/paimon/table/format/format_table.h b/include/paimon/table/format/format_table.h index a4c310254..d2db6aa4a 100644 --- a/include/paimon/table/format/format_table.h +++ b/include/paimon/table/format/format_table.h @@ -24,6 +24,7 @@ #include #include "paimon/catalog/identifier.h" +#include "paimon/catalog/table.h" #include "paimon/result.h" #include "paimon/schema/schema.h" #include "paimon/status.h" @@ -47,13 +48,18 @@ class FileSystem; /// Writes only insert - there is nowhere to record an update or a delete - so a read fills /// `_VALUE_KIND` with inserts throughout. /// -/// Reading and writing go through the entry points every other table uses: `TableScan`, -/// `TableRead`, `FileStoreWrite` and `FileStoreCommit`. Each recognises a format table from the -/// schema under the table path, or takes one already loaded through the `FormatTable` constructor -/// every context builder has, which is how a table whose schema lives in a metastore is reached. +/// A format table is a `Table`, so the operators it inherits create its scan, read, write and +/// commit from this table, with the context's options applied on top of the ones it carries. The +/// entry points every other table uses remain open too - `TableScan::Create()` and friends - each +/// of which recognises a format table from the schema under the table path, or takes one already +/// loaded through the `FormatTable` constructor every context builder has, which is how a table +/// whose schema lives in a metastore is reached. +/// +/// Every instance lives in a `shared_ptr`, which is what its operators hand to +/// `FormatTable::Copy()`. /// /// See `docs/source/user_guide/format_table.rst` for what is not supported yet. -class PAIMON_EXPORT FormatTable { +class PAIMON_EXPORT FormatTable : public Table, public std::enable_shared_from_this { public: /// Formats a format table's files can be in. enum class Format { @@ -72,7 +78,7 @@ class PAIMON_EXPORT FormatTable { /// /// This needs a schema file under the table directory, which a table created through /// `SchemaManager` or a file system catalog has. A table whose schema lives in a metastore has - /// none, and is loaded through `Catalog::GetFormatTable()` instead. + /// none, and is loaded through `Catalog::GetTable()` instead. /// /// @param file_system File system holding the table directory. /// @param table_path Root path of the table, which is also its data location. @@ -119,7 +125,7 @@ class PAIMON_EXPORT FormatTable { /// Directory the data files live in. const std::string& Location() const { - return location_; + return GetRootPath(); } /// Format of every data file in the directory. @@ -155,27 +161,16 @@ class PAIMON_EXPORT FormatTable { return options_; } - /// A name to identify this table. - std::string Name() const { - return identifier_.GetObjectName(); - } - - /// Full name of the table, database.tableName. - std::string FullName() const; - - /// Schema of the table, including its partition fields. + /// Schema of the table, including its partition fields. It hides `Table::LatestSchema()`, + /// which returns the same schema as the base every table sees; this one hands it back as the + /// `DataSchema` a format table is always created with. std::shared_ptr LatestSchema() const { - return schema_; + return std::static_pointer_cast(schema_); } /// Schema of the table as an arrow schema, including its partition fields. Result> GetArrowSchema() const; - /// File system holding the table directory. - std::shared_ptr GetFileSystem() const { - return file_system_; - } - /// Whether this table's own metadata lives under its location, as told by whoever loaded it. /// /// Only then are the `schema` and `branch` directories below the location table metadata @@ -185,6 +180,28 @@ class PAIMON_EXPORT FormatTable { return location_carries_paimon_metadata_; } + /// Creates a scan of this table from a finished scan context, refusing by name what a format + /// table cannot honour. The context's options are applied on top of the ones this table + /// carries, the precedence every context builder promises. + Result> NewScan(std::unique_ptr context) const override; + + /// Creates a read of this table from a finished read context, refusing by name what a format + /// table cannot honour. The context's options are applied on top of the ones this table + /// carries, the precedence every context builder promises. + Result> NewRead(std::unique_ptr context) const override; + + /// Creates a write of this table from a finished write context, refusing by name what a format + /// table cannot honour. The context's options are applied on top of the ones this table + /// carries, the precedence every context builder promises. + Result> NewWrite( + std::unique_ptr context) const override; + + /// Creates a commit of this table from a finished commit context, refusing by name what a + /// format table cannot honour. The context's options are applied on top of the ones this + /// table carries, the precedence every context builder promises. + Result> NewCommit( + std::unique_ptr context) const override; + private: FormatTable(const std::shared_ptr& file_system, const std::string& location, const Identifier& identifier, const std::shared_ptr& schema, @@ -192,10 +209,15 @@ class PAIMON_EXPORT FormatTable { const std::string& file_compression, const std::string& partition_default_name, bool partition_only_value_in_path, bool location_carries_paimon_metadata); - std::shared_ptr file_system_; - std::string location_; - Identifier identifier_; - std::shared_ptr schema_; + /// This table as a `shared_ptr`, which the operators above hand to `FormatTable::Copy()`. + /// + /// `shared_from_this()` on a const method hands back a `shared_ptr`, while + /// `Copy()` takes a mutable one; the cast is safe because `Copy()` never writes to the table + /// it is given. + std::shared_ptr SharedSelf() const { + return std::const_pointer_cast(shared_from_this()); + } + std::map options_; Format format_; std::string file_compression_; diff --git a/include/paimon/write_context.h b/include/paimon/write_context.h index e0a940eb0..58c0b3f97 100644 --- a/include/paimon/write_context.h +++ b/include/paimon/write_context.h @@ -178,7 +178,7 @@ class PAIMON_EXPORT WriteContextBuilder { /// /// There is no commit user: a format table keeps no snapshot to record one in. /// - /// @param table The format table to write to, as `Catalog::GetFormatTable()` hands it back. + /// @param table The format table to write to, as `Catalog::GetTable()` hands it back. explicit WriteContextBuilder(const std::shared_ptr& table); ~WriteContextBuilder(); diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt index f92450d39..ad3d65f90 100644 --- a/src/paimon/CMakeLists.txt +++ b/src/paimon/CMakeLists.txt @@ -452,7 +452,6 @@ set(PAIMON_CORE_SRCS core/table/format/format_table_commit.cpp core/table/format/format_table_file_store_commit.cpp core/table/format/format_table_file_store_write.cpp - core/table/format/format_table_loader.cpp core/table/format/format_table_read.cpp core/table/format/format_table_scan.cpp core/table/format/format_table_write.cpp diff --git a/src/paimon/core/catalog/catalog.cpp b/src/paimon/core/catalog/catalog.cpp index 978c20e9d..318125ec6 100644 --- a/src/paimon/core/catalog/catalog.cpp +++ b/src/paimon/core/catalog/catalog.cpp @@ -26,11 +26,9 @@ #include "fmt/format.h" #include "paimon/catalog_options.h" #include "paimon/common/utils/string_utils.h" -#include "paimon/core/catalog/catalog_utils.h" #include "paimon/core/catalog/file_system_catalog.h" #include "paimon/core/core_options.h" #include "paimon/schema/schema.h" -#include "paimon/table/format/format_table.h" #ifdef PAIMON_ENABLE_REST #include "paimon/rest/rest_catalog.h" #endif @@ -74,19 +72,4 @@ bool Catalog::SupportsVersionManagement() const { return false; } -Result> Catalog::GetFormatTable(const Identifier& identifier) const { - // Refused here rather than in each override, so every catalog rejects a system table the same - // way and in the same order: the database first, so that an identifier of "sys" is rejected - // without being parsed as a table name. - PAIMON_RETURN_NOT_OK(CatalogUtils::CheckNotSystemTable(identifier, "getFormatTable")); - return LoadFormatTable(identifier); -} - -Result> Catalog::LoadFormatTable(const Identifier& identifier) const { - // The location and the schema are read separately, and every directory below the location is - // table content: a catalog that knows better overrides this and says so. - return CatalogUtils::LoadFormatTableInTwoRequests(*this, identifier, - /*location_carries_paimon_metadata=*/false); -} - } // namespace paimon diff --git a/src/paimon/core/catalog/catalog_utils.cpp b/src/paimon/core/catalog/catalog_utils.cpp index 1f2a3800a..721942a88 100644 --- a/src/paimon/core/catalog/catalog_utils.cpp +++ b/src/paimon/core/catalog/catalog_utils.cpp @@ -31,7 +31,6 @@ #include "paimon/defs.h" #include "paimon/result.h" #include "paimon/schema/schema.h" -#include "paimon/table/format/format_table.h" namespace paimon { @@ -117,12 +116,6 @@ Status CatalogUtils::CheckManagedTableType(const Identifier& identifier, } PAIMON_ASSIGN_OR_RAISE(TableType table_type, TableTypeDefine::FromOptions(data_schema->Options())); - if (table_type == TableType::FORMAT_TABLE) { - return Status::Invalid( - fmt::format("Cannot open format table '{}' as a Table in '{}', please use " - "'Catalog::GetFormatTable' or 'FormatTable::Create'.", - identifier.ToString(), action)); - } // A materialized table is a managed table that also carries the SQL it materializes. if (table_type != TableType::TABLE && table_type != TableType::MATERIALIZED_TABLE) { const std::map& options = data_schema->Options(); @@ -136,17 +129,4 @@ Status CatalogUtils::CheckManagedTableType(const Identifier& identifier, return Status::OK(); } -Result> CatalogUtils::LoadFormatTableInTwoRequests( - const Catalog& catalog, const Identifier& identifier, bool location_carries_paimon_metadata) { - PAIMON_ASSIGN_OR_RAISE(std::string location, catalog.GetTableLocation(identifier)); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, catalog.LoadTableSchema(identifier)); - std::shared_ptr data_schema = std::dynamic_pointer_cast(schema); - if (data_schema == nullptr) { - return Status::Invalid(fmt::format("{} is not a data table, so it cannot be a format table", - identifier.GetFullName())); - } - return FormatTable::Create(catalog.GetFileSystem(), location, identifier, data_schema, - location_carries_paimon_metadata, /*dynamic_options=*/{}); -} - } // namespace paimon diff --git a/src/paimon/core/catalog/catalog_utils.h b/src/paimon/core/catalog/catalog_utils.h index 565a6aef5..23cb77203 100644 --- a/src/paimon/core/catalog/catalog_utils.h +++ b/src/paimon/core/catalog/catalog_utils.h @@ -66,32 +66,19 @@ class CatalogUtils { /// branch name, system table name) cannot be used as a single path component. static Status CheckValidTableName(const Identifier& identifier); - /// Fails when `schema` is not a table a `Table` can describe. + /// Fails when `schema` is not a table a managed `Table` can describe. /// - /// Only a managed table is one: a format table is loaded through `GetFormatTable()`, and the - /// remaining table types are stored differently and are not implemented here, so handing one - /// back as a `Table` would promise snapshots it never had. A schema that is not a data - /// table's, such as a system table's, passes. + /// Only a managed table is one: the remaining table types are stored differently and are + /// not implemented here, so handing one back as a `Table` would promise snapshots it never + /// had. A schema that is not a data table's, such as a system table's, passes. A format + /// table is served by the caller - `GetTable()` hands one back as the `FormatTable` it is - + /// so its schema only reaches this check by mistake and falls to the refusal below. /// /// @param action The call being refused, qualified as a caller would write it, e.g. /// `Catalog::GetTable`. It names the entry point in the message. static Status CheckManagedTableType(const Identifier& identifier, const std::shared_ptr& schema, const std::string& action); - - /// Loads `identifier` as a format table by reading its location and its schema separately. - /// - /// What every catalog can do, so it is what `Catalog::GetFormatTable()` falls back to. A - /// catalog that can get both from one response should not use it, since two requests can - /// disagree. - /// - /// @param location_carries_paimon_metadata Whether this catalog put the table's own metadata - /// under the table path, which is what tells a `schema` or `branch` directory below the - /// location from a partition directory of the same name. See - /// `FormatTable::LocationCarriesPaimonMetadata()`. - static Result> LoadFormatTableInTwoRequests( - const Catalog& catalog, const Identifier& identifier, - bool location_carries_paimon_metadata); }; } // namespace paimon diff --git a/src/paimon/core/catalog/file_system_catalog.cpp b/src/paimon/core/catalog/file_system_catalog.cpp index 87c89de8c..14293021e 100644 --- a/src/paimon/core/catalog/file_system_catalog.cpp +++ b/src/paimon/core/catalog/file_system_catalog.cpp @@ -34,7 +34,9 @@ #include "paimon/common/utils/string_utils.h" #include "paimon/core/catalog/catalog_utils.h" #include "paimon/core/core_options.h" +#include "paimon/core/options/table_type.h" #include "paimon/core/snapshot.h" +#include "paimon/core/table/file_store_table.h" #include "paimon/core/table/system/global_system_tables.h" #include "paimon/core/table/system/system_table.h" #include "paimon/core/table/system/system_table_schema.h" @@ -44,6 +46,7 @@ #include "paimon/fs/file_system.h" #include "paimon/logging.h" #include "paimon/result.h" +#include "paimon/table/format/format_table.h" namespace arrow { class Schema; @@ -343,7 +346,7 @@ Result> FileSystemCatalog::GetTable(const Identifier& ide PAIMON_ASSIGN_OR_RAISE(std::string data_table_name, identifier.GetDataTableName()); Identifier data_identifier(identifier.GetDatabaseName(), data_table_name); PAIMON_ASSIGN_OR_RAISE(std::string table_path, GetTableLocation(data_identifier)); - return std::make_shared(fs_, table_path, schema, identifier); + return std::make_shared(fs_, table_path, schema, identifier); } // The schema is read on the table's own branch: `root/branch/branch-` below the // table path, which `main` resolves to itself. `GetBranchNameOrDefault` folds any @@ -353,14 +356,27 @@ Result> FileSystemCatalog::GetTable(const Identifier& ide if (!latest_schema) { return Status::NotExist(fmt::format("{} not exist", identifier.ToString())); } + PAIMON_ASSIGN_OR_RAISE(std::string table_path, GetTableLocation(identifier)); + // The schema's `type` says what the table is. A format table is the same directory read as + // plain data, so the schema read above is all it takes to serve one: no manifest is looked + // for, and the checks a format table adds run in `FormatTable::Create`. + std::shared_ptr data_schema = checked_pointer_cast(*latest_schema); + PAIMON_ASSIGN_OR_RAISE(bool is_format_table, + TableTypeDefine::IsFormatTable(data_schema->Options())); + if (is_format_table) { + // This catalog keeps a table's own metadata under the table path, so a `schema` or + // `branch` directory there is metadata rather than table content. + return FormatTable::Create(fs_, table_path, identifier, data_schema, + /*location_carries_paimon_metadata=*/true, + /*dynamic_options=*/{}); + } // Only a managed table is a `Table`; describing another type as one would hand back // snapshots and manifests it never had, so the failure belongs to the call that opens the // table rather than at the first read. PAIMON_RETURN_NOT_OK(CatalogUtils::CheckManagedTableType(identifier, *latest_schema, "FileSystemCatalog::GetTable")); - PAIMON_ASSIGN_OR_RAISE(std::string table_path, GetTableLocation(identifier)); - return std::make_shared
(fs_, table_path, checked_pointer_cast(*latest_schema), - identifier); + return std::make_shared( + fs_, table_path, checked_pointer_cast(*latest_schema), identifier); } Status FileSystemCatalog::DropDatabase(const std::string& name, bool ignore_if_not_exists, @@ -579,10 +595,4 @@ Result> FileSystemCatalog::GetLatestSnapshot( return std::optional(snapshot->ToSnapshotInfo()); } -Result> FileSystemCatalog::LoadFormatTable( - const Identifier& identifier) const { - return CatalogUtils::LoadFormatTableInTwoRequests(*this, identifier, - /*location_carries_paimon_metadata=*/true); -} - } // namespace paimon diff --git a/src/paimon/core/catalog/file_system_catalog.h b/src/paimon/core/catalog/file_system_catalog.h index ccc45f666..fd4360755 100644 --- a/src/paimon/core/catalog/file_system_catalog.h +++ b/src/paimon/core/catalog/file_system_catalog.h @@ -71,13 +71,6 @@ class FileSystemCatalog : public Catalog { Result> GetLatestSnapshot(const Identifier& identifier, const std::string& branch) const override; - protected: - /// This catalog keeps a table's schema under the table's own path, so the `schema` and - /// `branch` directories there are metadata rather than table content. Nothing else differs, - /// so the location and the schema are still read separately. - Result> LoadFormatTable( - const Identifier& identifier) const override; - private: /// Fails when `db_name` cannot be used as a single path component, so that the returned /// path always stays under `warehouse`. diff --git a/src/paimon/core/catalog/file_system_catalog_test.cpp b/src/paimon/core/catalog/file_system_catalog_test.cpp index 51e7c642d..cfe361bae 100644 --- a/src/paimon/core/catalog/file_system_catalog_test.cpp +++ b/src/paimon/core/catalog/file_system_catalog_test.cpp @@ -629,7 +629,7 @@ TEST(FileSystemCatalogTest, TestCreateTableWithBlob) { ArrowSchemaRelease(&schema); } -TEST(FileSystemCatalogTest, TestGetTableRejectsFormatTable) { +TEST(FileSystemCatalogTest, TestGetTableServesAFormatTable) { std::map options; options[Options::FILE_SYSTEM] = "local"; options[Options::FILE_FORMAT] = "orc"; @@ -650,11 +650,11 @@ TEST(FileSystemCatalogTest, TestGetTableRejectsFormatTable) { /*ignore_if_exists=*/false)); ArrowSchemaRelease(&schema); - // A format table has no snapshots and no manifests, so handing it back as a Table would - // describe it as something it is not. - ASSERT_NOK_WITH_MSG(catalog.GetTable(Identifier("db1", "fmt")), "Cannot open format table"); - ASSERT_OK_AND_ASSIGN(std::shared_ptr format_table, - catalog.GetFormatTable(Identifier("db1", "fmt"))); + // The table's `type` says what it is, so the one call that opens a table hands a format + // table back as the `FormatTable` it is, through the `Table` interface the two share. + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, catalog.GetTable(Identifier("db1", "fmt"))); + std::shared_ptr format_table = std::dynamic_pointer_cast(table); + ASSERT_NE(nullptr, format_table); ASSERT_EQ(format_table->GetFormat(), FormatTable::Format::PARQUET); ASSERT_EQ(format_table->FullName(), "db1.fmt"); @@ -662,15 +662,6 @@ TEST(FileSystemCatalogTest, TestGetTableRejectsFormatTable) { // `branch` directories there are its metadata and a scan leaves them alone. A catalog that // keeps schemas elsewhere says no, and then every directory below the location is data. ASSERT_TRUE(format_table->LocationCarriesPaimonMetadata()); - - // A system table is refused by `Catalog::GetFormatTable()` itself, so every catalog answers - // alike. The database is checked before the table name is parsed, so a malformed name under - // "sys" is refused as a system table rather than surfacing a parse error. - ASSERT_NOK_WITH_MSG(catalog.GetFormatTable(Identifier("db1", "fmt$snapshots")), - "for system table"); - ASSERT_NOK_WITH_MSG(catalog.GetFormatTable(Identifier("sys", "all_table_options")), - "for system table"); - ASSERT_NOK_WITH_MSG(catalog.GetFormatTable(Identifier("sys", "a$b$c$d")), "for system table"); } TEST(FileSystemCatalogTest, TestInvalidCreateTable) { diff --git a/src/paimon/core/table/file_store_table.h b/src/paimon/core/table/file_store_table.h new file mode 100644 index 000000000..409fefca9 --- /dev/null +++ b/src/paimon/core/table/file_store_table.h @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include + +#include "paimon/catalog/identifier.h" +#include "paimon/catalog/table.h" +#include "paimon/result.h" +#include "paimon/type_fwd.h" + +namespace paimon { + +class Schema; + +/// The managed table a catalog hands out: one with snapshots and manifests, which the schema under +/// its root path describes. A format table - a directory of plain data files - is a `FormatTable`, +/// the other kind of `Table`. +/// +/// The operators create through the context-only entry points every table serves, so a context +/// built without a table reaches the same operators: `TableScan::Create()`, `TableRead::Create()`, +/// `FileStoreWrite::Create()` and `FileStoreCommit::Create()`, which read the table's schema to +/// dispatch on its type. +class FileStoreTable : public Table { + public: + /// The constructor a catalog builds a managed table through, so a `shared_ptr` can hold one; + /// the base constructor is protected, which an inherited constructor would stay. + /// @param file_system File system the table's data is accessed through. + /// @param root_path Root path of the table in the file system. + /// @param uuid Metastore UUID, or empty if unavailable. + FileStoreTable(const std::shared_ptr& file_system, const std::string& root_path, + const std::shared_ptr& schema, const Identifier& identifier, + const std::string& uuid = "") + : Table(file_system, root_path, schema, identifier, uuid) {} + + Result> NewScan(std::unique_ptr context) const override; + + Result> NewRead(std::unique_ptr context) const override; + + Result> NewWrite( + std::unique_ptr context) const override; + + Result> NewCommit( + std::unique_ptr context) const override; +}; + +} // namespace paimon diff --git a/src/paimon/core/table/format/format_table.cpp b/src/paimon/core/table/format/format_table.cpp index a039baa4b..53d53b967 100644 --- a/src/paimon/core/table/format/format_table.cpp +++ b/src/paimon/core/table/format/format_table.cpp @@ -18,9 +18,15 @@ #include "paimon/table/format/format_table.h" +#include +#include #include +#include +#include +#include #include "fmt/format.h" +#include "paimon/commit_context.h" #include "paimon/common/utils/checked_cast.h" #include "paimon/common/utils/options_utils.h" #include "paimon/common/utils/string_utils.h" @@ -29,8 +35,20 @@ #include "paimon/core/schema/schema_manager.h" #include "paimon/core/schema/schema_validation.h" #include "paimon/core/schema/table_schema.h" +#include "paimon/core/table/format/format_table_file_store_commit.h" +#include "paimon/core/table/format/format_table_file_store_write.h" +#include "paimon/core/table/format/format_table_read.h" +#include "paimon/core/table/format/format_table_scan.h" #include "paimon/defs.h" +#include "paimon/file_store_commit.h" +#include "paimon/file_store_write.h" #include "paimon/fs/file_system.h" +#include "paimon/read_context.h" +#include "paimon/scan_context.h" +#include "paimon/status.h" +#include "paimon/table/source/table_read.h" +#include "paimon/table/source/table_scan.h" +#include "paimon/write_context.h" namespace paimon { @@ -45,10 +63,7 @@ FormatTable::FormatTable(const std::shared_ptr& file_system, const std::string& file_compression, const std::string& partition_default_name, bool partition_only_value_in_path, bool location_carries_paimon_metadata) - : file_system_(file_system), - location_(location), - identifier_(identifier), - schema_(schema), + : Table(file_system, location, schema, identifier), options_(options), format_(format), file_compression_(file_compression), @@ -192,20 +207,180 @@ Result> FormatTable::Copy( for (const auto& [key, value] : dynamic_options) { options[key] = value; } - return Create(table->file_system_, table->location_, table->identifier_, table->schema_, + return Create(table->file_system_, table->root_path_, table->identifier_, + std::static_pointer_cast(table->schema_), table->location_carries_paimon_metadata_, options); } const std::vector& FormatTable::PartitionKeys() const { - return schema_->PartitionKeys(); -} - -std::string FormatTable::FullName() const { - return identifier_.GetFullName(); + return LatestSchema()->PartitionKeys(); } Result> FormatTable::GetArrowSchema() const { return schema_->GetArrowSchema(); } +Result> FormatTable::NewScan( + std::unique_ptr context) const { + std::shared_ptr shared_context = std::move(context); + if (shared_context == nullptr) { + return Status::Invalid("scan context is null pointer"); + } + if (shared_context->GetMemoryPool() == nullptr) { + return Status::Invalid("memory pool is null pointer"); + } + if (shared_context->GetExecutor() == nullptr) { + return Status::Invalid("executor is null pointer"); + } + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table, + FormatTable::Copy(SharedSelf(), shared_context->GetOptions())); + // Anything a `ScanContext` carries that a format table cannot honour is refused rather than + // silently dropped. + if (shared_context->IsStreamingMode()) { + return Status::NotImplemented( + "a format table has no snapshots, so there is nothing for a streaming scan to follow"); + } + if (shared_context->GetRealtimeContext() != nullptr) { + return Status::NotImplemented( + "a format table has no real-time store to union with what is on disk"); + } + if (shared_context->GetGlobalIndexResult() != nullptr) { + return Status::NotImplemented("a format table carries no global index"); + } + std::map partition_filter; + std::shared_ptr filters = shared_context->GetScanFilters(); + if (filters != nullptr) { + if (filters->GetPredicate() != nullptr) { + return Status::NotImplemented( + "a format table scan does not take a predicate: its files carry no statistics to " + "skip by, and pruning by the partition columns a predicate names is not " + "implemented yet; give the partition values as a partition filter instead"); + } + if (filters->GetBucketFilter()) { + return Status::NotImplemented("a format table has no buckets to filter by"); + } + const std::vector>& partition_filters = + filters->GetPartitionFilters(); + if (partition_filters.size() > 1) { + return Status::NotImplemented("a format table scan takes at most one partition filter"); + } + if (partition_filters.size() == 1) { + partition_filter = partition_filters.front(); + } + } + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr scan, + FormatTableScan::Create(table, partition_filter, shared_context->GetLimit())); + return std::unique_ptr(std::move(scan)); +} + +Result> FormatTable::NewRead( + std::unique_ptr context) const { + std::shared_ptr read_context = std::move(context); + if (read_context == nullptr) { + return Status::Invalid("read context is null pointer"); + } + if (read_context->GetMemoryPool() == nullptr) { + return Status::Invalid("memory pool is null pointer"); + } + if (read_context->GetExecutor() == nullptr) { + return Status::Invalid("executor is null pointer"); + } + if (read_context->HasVectorOrFullTextSearch()) { + return Status::NotImplemented( + "VectorSearch or FullTextSearch is not supported for format tables"); + } + if (PredicateUtils::ContainsFullTextMatch(read_context->GetPredicate())) { + return Status::NotImplemented("FullTextMatch predicate execution is not implemented yet"); + } + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table, + FormatTable::Copy(SharedSelf(), read_context->GetOptions())); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read, + FormatTableRead::Create(table, read_context)); + return std::unique_ptr(std::move(read)); +} + +Result> FormatTable::NewWrite( + std::unique_ptr context) const { + if (context == nullptr) { + return Status::Invalid("write context is null pointer"); + } + if (context->GetMemoryPool() == nullptr) { + return Status::Invalid("memory pool is null pointer"); + } + if (context->GetExecutor() == nullptr) { + return Status::Invalid("executor is null pointer"); + } + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table, + FormatTable::Copy(SharedSelf(), context->GetOptions())); + if (context->GetCatalog() != nullptr) { + return Status::Invalid( + "WithCatalog() requires a native table; use WriteContextBuilder(FormatTable) for a " + "format table"); + } + if (context->IsStreamingMode()) { + return Status::NotImplemented( + "a format table has no snapshots, so there is nothing a streaming write could " + "commit against"); + } + if (context->GetRealtimeContext() != nullptr) { + return Status::NotImplemented("a format table has no real-time store to write into"); + } + if (!context->GetWriteSchema().empty()) { + return Status::NotImplemented( + "a format table write takes the table's own columns; a write schema naming a " + "subset of them is not supported yet"); + } + // A write id prefixes a postpone-bucket writer's files so that one compaction reader can + // put them back in order. A format table has no buckets, so it would identify nothing. + if (context->GetWriteId().has_value()) { + return Status::NotImplemented( + "a format table has no buckets, so a write id would name nothing"); + } + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr format_write, + FormatTableFileStoreWrite::Create(table, context->GetMemoryPool())); + return std::unique_ptr(std::move(format_write)); +} + +Result> FormatTable::NewCommit( + std::unique_ptr context) const { + if (context == nullptr) { + return Status::Invalid("commit context is null pointer"); + } + if (context->GetMemoryPool() == nullptr) { + return Status::Invalid("memory pool is null pointer"); + } + if (context->GetExecutor() == nullptr) { + return Status::Invalid("executor is null pointer"); + } + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr given_table, + FormatTable::Copy(SharedSelf(), context->GetOptions())); + if (!context->IgnoreEmptyCommit()) { + return Status::NotImplemented( + "a format table cannot record an empty commit: keeping one means writing a " + "snapshot that adds no files, and there are no snapshots here"); + } + if (context->UseRESTCatalogCommit()) { + return Status::NotImplemented( + "a format table commits by renaming files into place, not by sending a snapshot " + "to a rest catalog"); + } + if (context->GetTableId()) { + return Status::NotImplemented( + "a format table sends no commit table request, so there is nowhere to put a table id"); + } + if (context->GetCatalog() != nullptr) { + return Status::NotImplemented( + "a format table keeps no snapshot for a catalog to take, so it commits by renaming " + "files into place rather than through the catalog it was loaded from"); + } + if (context->AppendCommitCheckConflict()) { + return Status::NotImplemented( + "a format table has no manifests to check a concurrent commit against"); + } + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr format_commit, + FormatTableFileStoreCommit::Create(given_table)); + return std::unique_ptr(std::move(format_commit)); +} + } // namespace paimon diff --git a/src/paimon/core/table/format/format_table_loader.cpp b/src/paimon/core/table/format/format_table_loader.cpp deleted file mode 100644 index 797bc3351..000000000 --- a/src/paimon/core/table/format/format_table_loader.cpp +++ /dev/null @@ -1,87 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "paimon/core/table/format/format_table_loader.h" - -#include -#include - -#include "fmt/format.h" -#include "paimon/catalog/identifier.h" -#include "paimon/common/utils/checked_cast.h" -#include "paimon/common/utils/path_util.h" -#include "paimon/core/options/table_type.h" -#include "paimon/core/schema/schema_manager.h" -#include "paimon/core/schema/table_schema.h" -#include "paimon/core/utils/branch_manager.h" -#include "paimon/schema/schema.h" -#include "paimon/table/format/format_table.h" - -namespace paimon { - -Result> FormatTableLoader::TryLoad( - const std::shared_ptr& file_system, const std::string& table_path, - const std::string& branch, const std::map& options, - const std::optional& specific_table_schema, const SchemaManager* schema_manager, - std::shared_ptr* table_schema_out) { - assert(table_schema_out != nullptr); - table_schema_out->reset(); - - std::shared_ptr table_schema; - if (branch == BranchManager::DEFAULT_MAIN_BRANCH && specific_table_schema) { - // Handing the schema over only saves reading it. A context always names a table path and - // this library writes the metadata under it, so `schema` and `branch` stay metadata. - PAIMON_ASSIGN_OR_RAISE(table_schema, - TableSchema::CreateFromJson(specific_table_schema.value())); - } else { - // Through the caller's manager when it has one, so that the read warms the cache it goes - // on to use rather than a cache that dies with this call. - SchemaManager own_schema_manager(file_system, table_path, branch); - const SchemaManager& reader = - schema_manager != nullptr ? *schema_manager : own_schema_manager; - PAIMON_ASSIGN_OR_RAISE(std::optional> latest_schema, - reader.Latest()); - if (!latest_schema) { - return std::shared_ptr(); - } - table_schema = *latest_schema; - } - *table_schema_out = table_schema; - - // From the schema alone: see `FormatTable::Create()`. - PAIMON_ASSIGN_OR_RAISE(bool is_format_table, - TableTypeDefine::IsFormatTable(table_schema->Options())); - if (!is_format_table) { - return std::shared_ptr(); - } - // A format table has no branches, so it is refused rather than served from the table path - // under another branch's name. See `SchemaValidation::ValidateFormatTableSchema()`. - if (BranchManager::NormalizeBranch(branch) != BranchManager::DEFAULT_MAIN_BRANCH) { - return Status::Invalid( - fmt::format("a format table has no branches, so {} cannot be read on branch '{}'", - table_path, branch)); - } - // A context names a path, never a catalog identifier, so the table is known by the directory - // it sits in. - Identifier identifier(Identifier::kUnknownDatabase, PathUtil::GetName(table_path)); - return FormatTable::Create(file_system, table_path, identifier, - checked_pointer_cast(table_schema), - /*location_carries_paimon_metadata=*/true, options); -} - -} // namespace paimon diff --git a/src/paimon/core/table/format/format_table_loader.h b/src/paimon/core/table/format/format_table_loader.h deleted file mode 100644 index 290ee94a7..000000000 --- a/src/paimon/core/table/format/format_table_loader.h +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include -#include -#include -#include - -#include "paimon/result.h" - -namespace paimon { - -class FileSystem; -class FormatTable; -class SchemaManager; -class TableSchema; - -/// Recognises a format table behind a table path. -/// -/// The scan, read, write and commit entry points all take a path and have to answer the same -/// question before they can dispatch, so one place answers it and they cannot disagree. -class FormatTableLoader { - public: - FormatTableLoader() = delete; - ~FormatTableLoader() = delete; - - /// Loads `table_path` as a format table, or returns null when it is not one. - /// - /// Null is also the answer when there is no schema under the path at all: what to say about a - /// table that is not there is the managed path's to decide, and it says it in more detail. - /// - /// The table type is read from the schema, never from `options`: `type` is structural, and one - /// call's options must not decide what kind of table this is. - /// - /// @param branch Branch the schema is read from. Take it from the same place the managed path - /// in the same entry point takes it, or the two dispatch on different schemas. Anything - /// but the main branch is refused once the schema turns out to be a format table's, - /// which has no branches to read. - /// @param options Options given at the call, which win over the ones the schema stored. - /// @param specific_table_schema A schema the caller already holds, as json, which saves - /// reading one from under the path. Used only on the main branch, as on the managed - /// read path. - /// @param schema_manager The manager to read the schema through, or null to use one of this - /// call's own. A caller that has one must pass it: a manager caches what it read. - /// @param table_schema_out Set to the schema this read, or null when there is none, so that a - /// caller falling back to the managed path does not read it twice. - static Result> TryLoad( - const std::shared_ptr& file_system, const std::string& table_path, - const std::string& branch, const std::map& options, - const std::optional& specific_table_schema, - const SchemaManager* schema_manager, std::shared_ptr* table_schema_out); -}; - -} // namespace paimon diff --git a/src/paimon/core/table/format/format_table_test.cpp b/src/paimon/core/table/format/format_table_test.cpp index e8391f68c..2430ffbda 100644 --- a/src/paimon/core/table/format/format_table_test.cpp +++ b/src/paimon/core/table/format/format_table_test.cpp @@ -34,6 +34,7 @@ #include "arrow/c/bridge.h" #include "gtest/gtest.h" #include "paimon/cache/cache.h" +#include "paimon/catalog/table.h" #include "paimon/commit_context.h" #include "paimon/commit_message.h" #include "paimon/common/table/special_fields.h" @@ -58,6 +59,7 @@ #include "paimon/memory/memory_pool.h" #include "paimon/predicate/literal.h" #include "paimon/predicate/predicate_builder.h" +#include "paimon/predicate/vector_search.h" #include "paimon/read_context.h" #include "paimon/record_batch.h" #include "paimon/scan_context.h" @@ -2843,6 +2845,159 @@ TEST(FormatTableTest, TestTheGenericEntryPointsTakeALoadedFormatTable) { ASSERT_EQ(rows, 3); } +TEST(FormatTableTest, TestATableServesAFormatTableThroughItsOwnEntryPoints) { + // `NewScan()` and its kin take the table they were called on as the one to serve, the way Java + // Paimon's `Table` serves both kinds through one `newReadBuilder()`, so a caller holding a + // format table as the `Table` a catalog hands out reaches it through them as well. The + // contexts name a path only and carry no format table, so it is the table itself that + // answers. + std::unique_ptr dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + std::shared_ptr
table; + ASSERT_OK_AND_ASSIGN(table, CreateTable(dir->GetFileSystem(), dir->Str(), {"dt"})); + + WriteContextBuilder write_builder(dir->Str(), "test-user"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr write, + table->NewWrite(std::move(write_context))); + ASSERT_OK_AND_ASSIGN(std::unique_ptr batch, + MakeBatch({1, 2}, {"alice", "bob"}, "20240101", {{"dt", "20240101"}})); + ASSERT_OK(write->Write(std::move(batch))); + ASSERT_OK_AND_ASSIGN(std::vector> messages, + write->PrepareCommit()); + + CommitContextBuilder commit_builder(dir->Str(), "test-user"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, + table->NewCommit(std::move(commit_context))); + ASSERT_OK(commit->Commit(messages)); + ASSERT_OK(write->Close()); + + ScanContextBuilder scan_builder(dir->Str()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, table->NewScan(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); + ASSERT_EQ(plan->Splits().size(), 1u); + + ReadContextBuilder read_builder(dir->Str()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read, table->NewRead(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, read->CreateReader(plan->Splits())); + std::vector rows; + while (true) { + ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch read_batch, reader->NextBatch()); + if (BatchReader::IsEofBatch(read_batch)) { + break; + } + ASSERT_OK_AND_ASSIGN(std::shared_ptr record_batch, + ImportBatch(read_batch)); + // `_VALUE_KIND` first, then the table's own columns, exactly as the generic entry points + // give them. + auto ids = checked_pointer_cast(record_batch->column(1)); + auto names = checked_pointer_cast(record_batch->column(2)); + auto dts = checked_pointer_cast(record_batch->column(3)); + for (int64_t i = 0; i < record_batch->num_rows(); i++) { + rows.push_back(std::to_string(ids->Value(i)) + "|" + names->GetString(i) + "|" + + dts->GetString(i)); + } + } + reader->Close(); + ASSERT_EQ(rows, (std::vector{"1|alice|20240101", "2|bob|20240101"})); +} + +TEST(FormatTableTest, TestTheTableEntryPointsRefuseWhatAFormatTableCannotHonour) { + // A refusal is a promise about what was not done, so the table's own entry points must make + // the same promises the generic ones do. Each context is handed to both, and the answer has + // to come out the same whichever way in the caller takes. + std::unique_ptr dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + ASSERT_OK_AND_ASSIGN(std::shared_ptr format_table, + CreateTable(dir->GetFileSystem(), dir->Str(), {"dt"})); + std::shared_ptr
table = format_table; + + // A context that never was is refused first of all, and by name, exactly as the generic + // entry points refuse one. + ASSERT_NOK_WITH_MSG(TableScan::Create(nullptr), "scan context is null pointer"); + ASSERT_NOK_WITH_MSG(table->NewScan(nullptr), "scan context is null pointer"); + ASSERT_NOK_WITH_MSG(TableRead::Create(nullptr), "read context is null pointer"); + ASSERT_NOK_WITH_MSG(table->NewRead(nullptr), "read context is null pointer"); + ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(nullptr), "write context is null pointer"); + ASSERT_NOK_WITH_MSG(table->NewWrite(nullptr), "write context is null pointer"); + ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(nullptr), "commit context is null pointer"); + ASSERT_NOK_WITH_MSG(table->NewCommit(nullptr), "commit context is null pointer"); + + // A streaming scan follows snapshots, which a format table keeps none of. + auto streaming_scan_context = [&dir]() -> Result> { + ScanContextBuilder builder(dir->Str()); + builder.WithStreamingMode(true); + return builder.Finish(); + }; + ASSERT_OK_AND_ASSIGN(std::unique_ptr streaming_context, streaming_scan_context()); + ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(streaming_context)), + "nothing for a streaming scan to follow"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr streaming_context_again, + streaming_scan_context()); + ASSERT_NOK_WITH_MSG(table->NewScan(std::move(streaming_context_again)), + "nothing for a streaming scan to follow"); + + // Vector search needs an index a format table never builds. + auto vector_read_context = [&dir]() -> Result> { + ReadContextBuilder builder(dir->Str()); + builder.SetVectorSearch(std::make_shared( + /*field_name=*/"name", /*limit=*/1, std::vector{0.0f}, /*pre_filter=*/nullptr, + /*predicate=*/nullptr, /*distance_type=*/std::nullopt, + /*options=*/std::map{})); + return builder.Finish(); + }; + ASSERT_OK_AND_ASSIGN(std::unique_ptr vector_context, vector_read_context()); + ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(vector_context)), + "not supported for format tables"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr vector_context_again, vector_read_context()); + ASSERT_NOK_WITH_MSG(table->NewRead(std::move(vector_context_again)), + "not supported for format tables"); + + // A catalog takes the snapshot a native table commits, and a format table has none to give. + auto catalog = std::make_shared(); + catalog->SetTableSchema(format_table->LatestSchema()); + + auto catalog_write_context = [&dir, &catalog]() -> Result> { + WriteContextBuilder builder(dir->Str(), "test-user"); + return builder.WithCatalog(catalog, Identifier("db", "tbl")).Finish(); + }; + ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog_write, catalog_write_context()); + ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(std::move(catalog_write)), + "use WriteContextBuilder(FormatTable)"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog_write_again, + catalog_write_context()); + ASSERT_NOK_WITH_MSG(table->NewWrite(std::move(catalog_write_again)), + "use WriteContextBuilder(FormatTable)"); + + auto catalog_commit_context = [&dir, &catalog]() -> Result> { + CommitContextBuilder builder(dir->Str(), "test-user"); + return builder.WithCatalog(catalog, Identifier("db", "fmt")).Finish(); + }; + ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog_commit, catalog_commit_context()); + ASSERT_NOK_WITH_MSG(FileStoreCommit::Create(std::move(catalog_commit)), + "keeps no snapshot for a catalog to take"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog_commit_again, + catalog_commit_context()); + ASSERT_NOK_WITH_MSG(table->NewCommit(std::move(catalog_commit_again)), + "keeps no snapshot for a catalog to take"); + + // A write id prefixes a postpone-bucket writer's files; a format table has no buckets. + auto write_id_context = [&dir]() -> Result> { + WriteContextBuilder builder(dir->Str(), "test-user"); + builder.WithWriteId(3); + return builder.Finish(); + }; + ASSERT_OK_AND_ASSIGN(std::unique_ptr write_id, write_id_context()); + ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(std::move(write_id)), + "a write id would name nothing"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr write_id_again, write_id_context()); + ASSERT_NOK_WITH_MSG(table->NewWrite(std::move(write_id_again)), + "a write id would name nothing"); +} + TEST(FormatTableTest, TestAContextBuiltFromAFormatTableRefusesASecondAnswer) { // The table already carries its schema and the file system it was loaded through, from a // source the context cannot see behind, so a second answer is refused rather than dropped. diff --git a/src/paimon/core/table/table.cpp b/src/paimon/core/table/table.cpp index 05751f92e..3b50d2e85 100644 --- a/src/paimon/core/table/table.cpp +++ b/src/paimon/core/table/table.cpp @@ -30,14 +30,17 @@ #include "arrow/c/bridge.h" #include "fmt/format.h" #include "paimon/catalog/catalog.h" +#include "paimon/catalog/identifier.h" #include "paimon/commit_context.h" #include "paimon/common/global_index/btree/btree_defs.h" #include "paimon/common/predicate/predicate_validator.h" #include "paimon/common/types/data_field.h" #include "paimon/common/utils/arrow/status_utils.h" #include "paimon/common/utils/binary_row_partition_computer.h" +#include "paimon/common/utils/checked_cast.h" #include "paimon/common/utils/fields_comparator.h" #include "paimon/common/utils/options_utils.h" +#include "paimon/common/utils/path_util.h" #include "paimon/common/utils/string_utils.h" #include "paimon/core/catalog/catalog_snapshot_commit.h" #include "paimon/core/catalog/catalog_utils.h" @@ -65,6 +68,7 @@ #include "paimon/core/operation/key_value_file_store_scan.h" #include "paimon/core/operation/key_value_file_store_write.h" #include "paimon/core/options/merge_engine.h" +#include "paimon/core/options/table_type.h" #include "paimon/core/postpone/postpone_bucket_file_store_write.h" #include "paimon/core/realtime/realtime_context_impl.h" #include "paimon/core/realtime/realtime_schema_layout.h" @@ -72,11 +76,7 @@ #include "paimon/core/schema/schema_validation.h" #include "paimon/core/schema/table_schema.h" #include "paimon/core/table/bucket_mode.h" -#include "paimon/core/table/format/format_table_file_store_commit.h" -#include "paimon/core/table/format/format_table_file_store_write.h" -#include "paimon/core/table/format/format_table_loader.h" -#include "paimon/core/table/format/format_table_read.h" -#include "paimon/core/table/format/format_table_scan.h" +#include "paimon/core/table/file_store_table.h" #include "paimon/core/table/source/abstract_table_scan.h" #include "paimon/core/table/source/append_only_split_generator.h" #include "paimon/core/table/source/append_only_table_read.h" @@ -130,6 +130,45 @@ class MergeFunctionWrapper; namespace { +/// Recognises a format table behind `table_path`, or returns null when it is not one. +/// +/// The scan, read, write and commit entry points below all take a path and have to answer the +/// same question before they can dispatch, so one place answers it and they cannot disagree. +/// Null is also the answer when no schema was found under the path at all: what to say about a +/// table that is not there is the managed path's to decide, and it says it in more detail. +/// +/// The table type is read from the schema, never from `options`: `type` is structural, and one +/// call's options must not decide what kind of table this is. `branch` names the branch the +/// schema was read from: anything but the main branch is refused once the schema turns out to be +/// a format table's, which has no branches to read. +Result> TryLoadFormatTable( + const std::shared_ptr& file_system, const std::string& table_path, + const std::string& branch, const std::map& options, + const std::shared_ptr& table_schema) { + if (table_schema == nullptr) { + return std::shared_ptr(); + } + // From the schema alone: see `FormatTable::Create()`. + PAIMON_ASSIGN_OR_RAISE(bool is_format_table, + TableTypeDefine::IsFormatTable(table_schema->Options())); + if (!is_format_table) { + return std::shared_ptr(); + } + // A format table has no branches, so it is refused rather than served from the table path + // under another branch's name. See `SchemaValidation::ValidateFormatTableSchema()`. + if (BranchManager::NormalizeBranch(branch) != BranchManager::DEFAULT_MAIN_BRANCH) { + return Status::Invalid( + fmt::format("a format table has no branches, so {} cannot be read on branch '{}'", + table_path, branch)); + } + // A context names a path, never a catalog identifier, so the table is known by the directory + // it sits in. + Identifier identifier(Identifier::kUnknownDatabase, PathUtil::GetName(table_path)); + return FormatTable::Create(file_system, table_path, identifier, + checked_pointer_cast(table_schema), + /*location_carries_paimon_metadata=*/true, options); +} + Status RestoreRealtimeCommittedProgress(const std::shared_ptr& realtime_context, const std::shared_ptr& snapshot_manager, const CoreOptions& options) { @@ -148,39 +187,6 @@ Status RestoreRealtimeCommittedProgress(const std::shared_ptr& return Status::OK(); } -/// Maps a `WriteContext` onto `FormatTableFileStoreWrite`, refusing by name what a format table -/// cannot honour rather than silently dropping it. -Result> NewFormatTableWrite( - const std::shared_ptr& table, const WriteContext& ctx) { - if (ctx.GetCatalog() != nullptr) { - return Status::Invalid( - "WithCatalog() requires a native table; use WriteContextBuilder(FormatTable) for a " - "format table"); - } - if (ctx.IsStreamingMode()) { - return Status::NotImplemented( - "a format table has no snapshots, so there is nothing a streaming write could " - "commit against"); - } - if (ctx.GetRealtimeContext() != nullptr) { - return Status::NotImplemented("a format table has no real-time store to write into"); - } - if (!ctx.GetWriteSchema().empty()) { - return Status::NotImplemented( - "a format table write takes the table's own columns; a write schema naming a " - "subset of them is not supported yet"); - } - // A write id prefixes a postpone-bucket writer's files so that one compaction reader can - // put them back in order. A format table has no buckets, so it would identify nothing. - if (ctx.GetWriteId().has_value()) { - return Status::NotImplemented( - "a format table has no buckets, so a write id would name nothing"); - } - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr format_write, - FormatTableFileStoreWrite::Create(table, ctx.GetMemoryPool())); - return std::unique_ptr(std::move(format_write)); -} - Result HasHistoricalPrimaryKeyBTreeDefinition( const std::shared_ptr& schema_manager, int64_t current_schema_id) { if (current_schema_id <= TableSchema::FIRST_SCHEMA_ID) { @@ -214,20 +220,22 @@ Table::Table(const std::shared_ptr& file_system, const std::string& identifier_(identifier), uuid_(uuid) {} -Result> Table::NewScan(std::unique_ptr context) const { +Result> FileStoreTable::NewScan( + std::unique_ptr context) const { return TableScan::Create(std::move(context)); } -Result> Table::NewRead(std::unique_ptr context) const { +Result> FileStoreTable::NewRead( + std::unique_ptr context) const { return TableRead::Create(std::move(context)); } -Result> Table::NewWrite( +Result> FileStoreTable::NewWrite( std::unique_ptr context) const { return FileStoreWrite::Create(std::move(context)); } -Result> Table::NewCommit( +Result> FileStoreTable::NewCommit( std::unique_ptr context) const { return FileStoreCommit::Create(std::move(context)); } @@ -245,9 +253,7 @@ Result> FileStoreWrite::Create(std::unique_ptrGetFormatTable() != nullptr) { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr given_table, - FormatTable::Copy(ctx->GetFormatTable(), ctx->GetOptions())); - return NewFormatTableWrite(given_table, *ctx); + return ctx->GetFormatTable()->NewWrite(std::move(ctx)); } PAIMON_RETURN_NOT_OK(CheckVersionManagementImplemented(ctx->GetCatalog())); @@ -292,16 +298,28 @@ Result> FileStoreWrite::Create(std::unique_ptr(tmp_options.GetFileSystem(), ctx->GetRootPath(), branch); std::shared_ptr latest_schema; - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr format_table, - FormatTableLoader::TryLoad(tmp_options.GetFileSystem(), ctx->GetRootPath(), branch, - ctx->GetOptions(), catalog_table_schema, schema_manager.get(), - &latest_schema)); + if (branch == BranchManager::DEFAULT_MAIN_BRANCH && catalog_table_schema) { + // Handing the schema over only saves reading it. A context always names a table path and + // this library writes the metadata under it, so `schema` and `branch` stay metadata. + PAIMON_ASSIGN_OR_RAISE(latest_schema, + TableSchema::CreateFromJson(catalog_table_schema.value())); + } else { + // Through the manager built above, so the read warms the cache the managed path below + // goes on to use. + PAIMON_ASSIGN_OR_RAISE(std::optional> latest, + schema_manager->Latest()); + if (latest) { + latest_schema = *latest; + } + } + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr format_table, + TryLoadFormatTable(tmp_options.GetFileSystem(), ctx->GetRootPath(), + branch, ctx->GetOptions(), latest_schema)); if (format_table != nullptr) { - return NewFormatTableWrite(format_table, *ctx); + return format_table->NewWrite(std::move(ctx)); } - // The schema the dispatch above already read through `schema_manager`, rather than a second - // read of the same file. + // The schema the branch read above through `schema_manager`, rather than a second read of + // the same file. if (latest_schema == nullptr) { return Status::Invalid(fmt::format("not found latest schema in branch {}", branch)); } @@ -678,53 +696,6 @@ class TableScanImpl { Result> NewDataTableScan( const std::shared_ptr& context, const std::shared_ptr& loaded_schema); -/// Maps a `ScanContext` onto `FormatTableScan`, which takes one partition filter and a limit and -/// nothing else. -/// -/// A format table has no buckets, so a bucket filter has nothing to act on. A predicate is -/// refused too, which is narrower than Java: `FormatReadBuilder.newScan()` hands the partition -/// half of one to the scan to prune directories with, which this does not do yet. -Result> NewFormatTableScan(const std::shared_ptr& table, - const std::shared_ptr& context) { - // Anything a `ScanContext` carries that a format table cannot honour is refused rather than - // silently dropped. - if (context->IsStreamingMode()) { - return Status::NotImplemented( - "a format table has no snapshots, so there is nothing for a streaming scan to follow"); - } - if (context->GetRealtimeContext() != nullptr) { - return Status::NotImplemented( - "a format table has no real-time store to union with what is on disk"); - } - if (context->GetGlobalIndexResult() != nullptr) { - return Status::NotImplemented("a format table carries no global index"); - } - std::map partition_filter; - std::shared_ptr filters = context->GetScanFilters(); - if (filters != nullptr) { - if (filters->GetPredicate() != nullptr) { - return Status::NotImplemented( - "a format table scan does not take a predicate: its files carry no statistics to " - "skip by, and pruning by the partition columns a predicate names is not " - "implemented yet; give the partition values as a partition filter instead"); - } - if (filters->GetBucketFilter()) { - return Status::NotImplemented("a format table has no buckets to filter by"); - } - const std::vector>& partition_filters = - filters->GetPartitionFilters(); - if (partition_filters.size() > 1) { - return Status::NotImplemented("a format table scan takes at most one partition filter"); - } - if (partition_filters.size() == 1) { - partition_filter = partition_filters.front(); - } - } - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan, - FormatTableScan::Create(table, partition_filter, context->GetLimit())); - return std::unique_ptr(std::move(scan)); -} - } // namespace Result> TableScan::Create(std::unique_ptr context) { @@ -738,43 +709,49 @@ Result> TableScan::Create(std::unique_ptr shared_context = std::move(context); // A table the caller already loaded says what it is, so nothing is read to find out. - if (shared_context->GetFormatTable() != nullptr) { - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr format_table, - FormatTable::Copy(shared_context->GetFormatTable(), shared_context->GetOptions())); - return NewFormatTableScan(format_table, shared_context); + if (context->GetFormatTable() != nullptr) { + return context->GetFormatTable()->NewScan(std::move(context)); } // load schema - PAIMON_ASSIGN_OR_RAISE(CoreOptions tmp_options, - CoreOptions::FromMap(shared_context->GetOptions(), - shared_context->GetSpecificFileSystem(), {})); + PAIMON_ASSIGN_OR_RAISE( + CoreOptions tmp_options, + CoreOptions::FromMap(context->GetOptions(), context->GetSpecificFileSystem(), {})); PAIMON_ASSIGN_OR_RAISE(std::optional system_table_path, - SystemTableLoader::TryParsePath(shared_context->GetPath())); + SystemTableLoader::TryParsePath(context->GetPath())); if (system_table_path) { PAIMON_ASSIGN_OR_RAISE( std::shared_ptr system_table, - SystemTableLoader::LoadFromPath(tmp_options.GetFileSystem(), shared_context->GetPath(), - shared_context->GetOptions())); - return system_table->NewScan(shared_context); + SystemTableLoader::LoadFromPath(tmp_options.GetFileSystem(), context->GetPath(), + context->GetOptions())); + return system_table->NewScan(std::move(context)); } // A format table is planned by listing directories, so it never reaches the manifest path // below. One `TableScan` interface serves both, as Java Paimon serves both through one // `ReadBuilder`. + const std::string branch = BranchManager::NormalizeBranch(tmp_options.GetBranch()); std::shared_ptr latest_schema; - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr format_table, - FormatTableLoader::TryLoad(tmp_options.GetFileSystem(), shared_context->GetPath(), - BranchManager::NormalizeBranch(tmp_options.GetBranch()), - shared_context->GetOptions(), - shared_context->GetSpecificTableSchema(), - /*schema_manager=*/nullptr, &latest_schema)); + if (branch == BranchManager::DEFAULT_MAIN_BRANCH && context->GetSpecificTableSchema()) { + // Handing the schema over only saves reading it. A context always names a table path and + // this library writes the metadata under it, so `schema` and `branch` stay metadata. + PAIMON_ASSIGN_OR_RAISE( + latest_schema, TableSchema::CreateFromJson(context->GetSpecificTableSchema().value())); + } else { + SchemaManager schema_manager(tmp_options.GetFileSystem(), context->GetPath(), branch); + PAIMON_ASSIGN_OR_RAISE(std::optional> latest, + schema_manager.Latest()); + if (latest) { + latest_schema = *latest; + } + } + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr format_table, + TryLoadFormatTable(tmp_options.GetFileSystem(), context->GetPath(), + branch, context->GetOptions(), latest_schema)); if (format_table != nullptr) { - return NewFormatTableScan(format_table, shared_context); + return format_table->NewScan(std::move(context)); } - // With the schema the dispatch already read, so the managed path does not read it again. - return NewDataTableScan(shared_context, latest_schema); + // With the schema already read above, so the managed path does not read it again. + return NewDataTableScan(std::move(context), latest_schema); } namespace { @@ -821,29 +798,16 @@ Status ValidateRealtimeScan(const TableSchema& table_schema, const CoreOptions& return Status::OK(); } +/// @param loaded_schema The schema `TableScan::Create()` already read while dispatching on the +/// table type, or null when neither the caller's json nor the branch's own directory had +/// one. Result> NewDataTableScan( const std::shared_ptr& context, const std::shared_ptr& loaded_schema) { - PAIMON_ASSIGN_OR_RAISE( - CoreOptions tmp_options, - CoreOptions::FromMap(context->GetOptions(), context->GetSpecificFileSystem(), {})); - std::string branch = BranchManager::NormalizeBranch(tmp_options.GetBranch()); - std::shared_ptr table_schema; - const auto& specific_table_schema = context->GetSpecificTableSchema(); - if (loaded_schema != nullptr) { - table_schema = loaded_schema; - } else if (branch == BranchManager::DEFAULT_MAIN_BRANCH && specific_table_schema) { - PAIMON_ASSIGN_OR_RAISE(table_schema, - TableSchema::CreateFromJson(specific_table_schema.value())); - } else { - SchemaManager schema_manager(tmp_options.GetFileSystem(), context->GetPath(), branch); - PAIMON_ASSIGN_OR_RAISE(std::optional> latest_table_schema, - schema_manager.Latest()); - if (latest_table_schema == std::nullopt) { - return Status::Invalid("not found latest schema"); - } - table_schema = latest_table_schema.value(); + if (loaded_schema == nullptr) { + return Status::Invalid("not found latest schema"); } + const std::shared_ptr& table_schema = loaded_schema; PAIMON_ASSIGN_OR_RAISE(bool read_optimized, OptionsUtils::GetValueFromMap(context->GetOptions(), kReadOptimizedScanOption, false)); @@ -1053,84 +1017,69 @@ Result> NewDataTableRead( std::move(fallback_table_read), arrow_pool); } -/// Maps a `ReadContext` onto `FormatTableRead`, which reads everything it needs out of the context -/// itself: the columns, the predicate, the pool and executor, and what a file is opened with. It -/// refuses by name what a format table cannot honour. -Result> NewFormatTableRead(const std::shared_ptr& table, - const std::shared_ptr& context) { - if (PredicateUtils::ContainsFullTextMatch(context->GetPredicate())) { - return Status::NotImplemented("FullTextMatch predicate execution is not implemented yet"); - } - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read, - FormatTableRead::Create(table, context)); - return std::unique_ptr(std::move(read)); -} - } // namespace Result> TableRead::Create(std::unique_ptr ctx) { - std::shared_ptr context = std::move(ctx); - if (context == nullptr) { + if (ctx == nullptr) { return Status::Invalid("read context is null pointer"); } - if (context->GetMemoryPool() == nullptr) { + if (ctx->GetMemoryPool() == nullptr) { return Status::Invalid("memory pool is null pointer"); } - if (context->GetExecutor() == nullptr) { + if (ctx->GetExecutor() == nullptr) { return Status::Invalid("executor is null pointer"); } // A table the caller already loaded says what it is, so nothing is read to find out. - if (context->GetFormatTable() != nullptr) { - if (context->HasVectorOrFullTextSearch()) { - return Status::NotImplemented( - "VectorSearch or FullTextSearch is not supported for format tables"); - } - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr format_table, - FormatTable::Copy(context->GetFormatTable(), context->GetOptions())); - return NewFormatTableRead(format_table, context); + if (ctx->GetFormatTable() != nullptr) { + return ctx->GetFormatTable()->NewRead(std::move(ctx)); } - PAIMON_ASSIGN_OR_RAISE( - CoreOptions tmp_core_options, - CoreOptions::FromMap(context->GetOptions(), context->GetSpecificFileSystem(), - context->GetFileSystemSchemeToIdentifierMap())); + PAIMON_ASSIGN_OR_RAISE(CoreOptions tmp_core_options, + CoreOptions::FromMap(ctx->GetOptions(), ctx->GetSpecificFileSystem(), + ctx->GetFileSystemSchemeToIdentifierMap())); PAIMON_ASSIGN_OR_RAISE(std::optional system_table_path, - SystemTableLoader::TryParsePath(context->GetPath())); + SystemTableLoader::TryParsePath(ctx->GetPath())); if (system_table_path) { - if (PredicateUtils::ContainsFullTextMatch(context->GetPredicate())) { + if (PredicateUtils::ContainsFullTextMatch(ctx->GetPredicate())) { return Status::NotImplemented( "FullTextMatch predicate execution is not implemented yet"); } - if (context->HasVectorOrFullTextSearch()) { + if (ctx->HasVectorOrFullTextSearch()) { return Status::NotImplemented( "VectorSearch or FullTextSearch is not supported for system tables"); } - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr system_table, - SystemTableLoader::LoadFromPath(tmp_core_options.GetFileSystem(), context->GetPath(), - context->GetOptions())); - return system_table->NewRead(context); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr system_table, + SystemTableLoader::LoadFromPath(tmp_core_options.GetFileSystem(), + ctx->GetPath(), ctx->GetOptions())); + return system_table->NewRead(std::move(ctx)); } // A format table has no manifest and a file another engine wrote carries no field ids, so // which files there are and how a row is put back together is its own. How a file is opened // is not: both paths go through `DataFileReaderFactory`. + const std::string branch = ctx->GetBranch(); std::shared_ptr latest_schema; - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr format_table, - FormatTableLoader::TryLoad(tmp_core_options.GetFileSystem(), context->GetPath(), - context->GetBranch(), context->GetOptions(), - context->GetSpecificTableSchema(), - /*schema_manager=*/nullptr, &latest_schema)); - if (format_table != nullptr) { - if (context->HasVectorOrFullTextSearch()) { - return Status::NotImplemented( - "VectorSearch or FullTextSearch is not supported for format tables"); + if (branch == BranchManager::DEFAULT_MAIN_BRANCH && ctx->GetSpecificTableSchema()) { + // Handing the schema over only saves reading it. A context always names a table path and + // this library writes the metadata under it, so `schema` and `branch` stay metadata. + PAIMON_ASSIGN_OR_RAISE(latest_schema, + TableSchema::CreateFromJson(ctx->GetSpecificTableSchema().value())); + } else { + SchemaManager schema_manager(tmp_core_options.GetFileSystem(), ctx->GetPath(), branch); + PAIMON_ASSIGN_OR_RAISE(std::optional> latest, + schema_manager.Latest()); + if (latest) { + latest_schema = *latest; } - return NewFormatTableRead(format_table, context); + } + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr format_table, + TryLoadFormatTable(tmp_core_options.GetFileSystem(), ctx->GetPath(), + branch, ctx->GetOptions(), latest_schema)); + if (format_table != nullptr) { + return format_table->NewRead(std::move(ctx)); } - // With the schema the dispatch already read, so the managed path does not read it again. - return NewDataTableRead(context, latest_schema); + // With the schema already read above, so the managed path does not read it again. + return NewDataTableRead(std::move(ctx), latest_schema); } namespace { @@ -1206,37 +1155,6 @@ Result> NewSnapshotCommit( std::make_shared(fs, snapshot_manager)); } -/// Creates a format-table commit, rejecting non-default options that require snapshots. -Result> NewFormatTableCommit( - const std::shared_ptr& table, const CommitContext& ctx) { - if (!ctx.IgnoreEmptyCommit()) { - return Status::NotImplemented( - "a format table cannot record an empty commit: keeping one means writing a " - "snapshot that adds no files, and there are no snapshots here"); - } - if (ctx.UseRESTCatalogCommit()) { - return Status::NotImplemented( - "a format table commits by renaming files into place, not by sending a snapshot " - "to a rest catalog"); - } - if (ctx.GetTableId()) { - return Status::NotImplemented( - "a format table sends no commit table request, so there is nowhere to put a table id"); - } - if (ctx.GetCatalog() != nullptr) { - return Status::NotImplemented( - "a format table keeps no snapshot for a catalog to take, so it commits by renaming " - "files into place rather than through the catalog it was loaded from"); - } - if (ctx.AppendCommitCheckConflict()) { - return Status::NotImplemented( - "a format table has no manifests to check a concurrent commit against"); - } - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr format_commit, - FormatTableFileStoreCommit::Create(table)); - return std::unique_ptr(std::move(format_commit)); -} - } // namespace Result> FileStoreCommit::Create( @@ -1253,9 +1171,7 @@ Result> FileStoreCommit::Create( // A table the caller already loaded says what it is, so nothing is read to find out. if (ctx->GetFormatTable() != nullptr) { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr given_table, - FormatTable::Copy(ctx->GetFormatTable(), ctx->GetOptions())); - return NewFormatTableCommit(given_table, *ctx); + return ctx->GetFormatTable()->NewCommit(std::move(ctx)); } PAIMON_RETURN_NOT_OK(CheckVersionManagementImplemented(ctx->GetCatalog())); @@ -1320,12 +1236,25 @@ Result> FileStoreCommit::Create( auto schema_manager = std::make_shared(tmp_options.GetFileSystem(), root_path, branch); std::shared_ptr latest_schema; + if (branch == BranchManager::DEFAULT_MAIN_BRANCH && catalog_table_schema) { + // Handing the schema over only saves reading it. A context always names a table path and + // this library writes the metadata under it, so `schema` and `branch` stay metadata. + PAIMON_ASSIGN_OR_RAISE(latest_schema, + TableSchema::CreateFromJson(catalog_table_schema.value())); + } else { + // Through the manager built above, so the read warms the cache the managed path below + // goes on to use. + PAIMON_ASSIGN_OR_RAISE(std::optional> latest, + schema_manager->Latest()); + if (latest) { + latest_schema = *latest; + } + } PAIMON_ASSIGN_OR_RAISE(std::shared_ptr format_table, - FormatTableLoader::TryLoad( - tmp_options.GetFileSystem(), root_path, branch, ctx->GetOptions(), - catalog_table_schema, schema_manager.get(), &latest_schema)); + TryLoadFormatTable(tmp_options.GetFileSystem(), root_path, branch, + ctx->GetOptions(), latest_schema)); if (format_table != nullptr) { - return NewFormatTableCommit(format_table, *ctx); + return format_table->NewCommit(std::move(ctx)); } if (latest_schema == nullptr) { return Status::Invalid(fmt::format("not found latest schema in branch {}", branch)); diff --git a/src/paimon/core/table/table_test.cpp b/src/paimon/core/table/table_test.cpp index 4792a8461..78d01e5a5 100644 --- a/src/paimon/core/table/table_test.cpp +++ b/src/paimon/core/table/table_test.cpp @@ -31,11 +31,13 @@ #include "paimon/core/catalog/file_system_catalog.h" #include "paimon/core/schema/schema_manager.h" #include "paimon/core/schema/table_schema.h" +#include "paimon/core/table/file_store_table.h" #include "paimon/file_store_commit.h" #include "paimon/file_store_write.h" #include "paimon/read_context.h" #include "paimon/scan_context.h" #include "paimon/schema/schema.h" +#include "paimon/table/format/format_table.h" #include "paimon/table/source/table_read.h" #include "paimon/table/source/table_scan.h" #include "paimon/testing/mock/mock_catalog.h" @@ -57,7 +59,7 @@ Result> CreateManagedTable(const std::shared_ptr created, schema_manager.CreateTable(schema, /*partition_keys=*/{}, /*primary_keys=*/{}, /*options=*/{})); - return std::make_shared
( + return std::make_shared( file_system, path, checked_pointer_cast(std::shared_ptr(std::move(created))), identifier); } @@ -164,7 +166,7 @@ TEST(TableTest, TestMainBranchAliasResolvesToTheMainBranch) { SCOPED_TRACE(main_identifier.ToString()); // A catalog that loaded the schema itself calls the constructor directly, which a // rest catalog is shaped like. The alias resolves to the main branch as well. - Table table(fs, dir->Str(), schema, main_identifier); + FileStoreTable table(fs, dir->Str(), schema, main_identifier); ASSERT_OK_AND_ASSIGN(std::string branch, table.CurrentBranch()); EXPECT_EQ("main", branch); } @@ -211,10 +213,9 @@ TEST(TableTest, TestCatalogOpenedTableServesThePerTableFileSystem) { EXPECT_EQ(opened->GetRootPath(), ""); } -TEST(TableTest, TestGetTableRejectsATableTypeThatIsNotAManagedTable) { - // A `Table` describes a managed paimon table. Handing back another type as one would promise - // snapshots and manifests it never had, so the failure belongs at the call that opens the - // table rather than at the first read. +TEST(TableTest, TestGetTableServesEachKindOfTableAsWhatItIs) { + // The table's `type` option says what it is, so the one call that opens a table hands each + // kind back as what it is, through the `Table` interface the two kinds share. auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); auto fs = dir->GetFileSystem(); @@ -230,7 +231,10 @@ TEST(TableTest, TestGetTableRejectsATableTypeThatIsNotAManagedTable) { format_schema_manager.CreateTable(schema, /*partition_keys=*/{"dt"}, /*primary_keys=*/{}, {{"type", "format-table"}, {"file.format", "parquet"}})); - ASSERT_NOK_WITH_MSG(catalog.GetTable(Identifier("db", "format")), "Cannot open format table"); + // A format table comes back as the `FormatTable` it is. + ASSERT_OK_AND_ASSIGN(std::shared_ptr
opened, + catalog.GetTable(Identifier("db", "format"))); + ASSERT_NE(nullptr, std::dynamic_pointer_cast(opened)); // A managed table with no `type` option at all is what a `Table` is for, and still opens. const std::string managed_dir = dir->Str() + "/db.db/managed"; diff --git a/src/paimon/rest/rest_catalog.cpp b/src/paimon/rest/rest_catalog.cpp index 66898d8af..fbd9521c4 100644 --- a/src/paimon/rest/rest_catalog.cpp +++ b/src/paimon/rest/rest_catalog.cpp @@ -32,8 +32,10 @@ #include "paimon/common/utils/string_utils.h" #include "paimon/core/catalog/catalog_utils.h" #include "paimon/core/core_options.h" +#include "paimon/core/options/table_type.h" #include "paimon/core/schema/schema_validation.h" #include "paimon/core/schema/table_schema.h" +#include "paimon/core/table/file_store_table.h" #include "paimon/core/table/system/global_system_tables.h" #include "paimon/core/table/system/system_table.h" #include "paimon/core/table/system/system_table_schema.h" @@ -430,36 +432,35 @@ Result> RestCatalog::LoadTableSchema(const Identifier& i return checked_pointer_cast(schema); } -Result> RestCatalog::LoadFormatTable( - const Identifier& identifier) const { - // A system table is already refused by `Catalog::GetFormatTable()`, which every caller goes - // through, so this only has to load. - PAIMON_ASSIGN_OR_RAISE(std::optional branch, identifier.GetBranchName()); - branch = NormalizeBranch(std::move(branch)); - PAIMON_ASSIGN_OR_RAISE(Identifier load_identifier, ToLoadIdentifier(identifier)); - std::string location; - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr schema, - LoadDataTableSchema(load_identifier, branch, &location, /*table_id=*/nullptr)); - // A rest catalog holds the schema itself, so everything below the location is data. - return FormatTable::Create(fs_, location, identifier, checked_pointer_cast(schema), - /*location_carries_paimon_metadata=*/false, - /*dynamic_options=*/{}); -} - Result> RestCatalog::GetTable(const Identifier& identifier) const { // Read UUID, path and schema together to avoid mixing table generations after recreation. std::string uuid; std::string table_path; PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, LoadTableSchema(identifier, &uuid, &table_path)); + // The schema the same response carried says what the table is: a format table is handed + // back as the `FormatTable` it is, built from the location of the same response, rather + // than refused as the managed table it is not. + std::shared_ptr data_schema = std::dynamic_pointer_cast(schema); + if (data_schema != nullptr) { + PAIMON_ASSIGN_OR_RAISE(bool is_format_table, + TableTypeDefine::IsFormatTable(data_schema->Options())); + if (is_format_table) { + // A rest catalog holds the schema itself, so everything below the location is data. + // The catalog-wide file system, as a format table reads no metadata of its own. + return FormatTable::Create(fs_, table_path, identifier, data_schema, + /*location_carries_paimon_metadata=*/false, + /*dynamic_options=*/{}); + } + } PAIMON_RETURN_NOT_OK( CatalogUtils::CheckManagedTableType(identifier, schema, "Catalog::GetTable")); // The table holds the file system its own data is served by, which a server issuing data // tokens binds to this table. PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table_file_system, GetTableFileSystem(identifier)); - return std::make_shared
(table_file_system, table_path, schema, identifier, uuid); + return std::make_shared(table_file_system, table_path, schema, identifier, + uuid); } std::string RestCatalog::GetRootPath() const { diff --git a/src/paimon/rest/rest_catalog.h b/src/paimon/rest/rest_catalog.h index 051b90d38..6b7f22cda 100644 --- a/src/paimon/rest/rest_catalog.h +++ b/src/paimon/rest/rest_catalog.h @@ -110,12 +110,6 @@ class RestCatalog : public Catalog, public VersionManagedCatalog { /// Options merged with the server side config. const std::map& GetOptions() const override; - protected: - /// Loads the location and the schema from one `GetTable` response, so the table cannot be - /// built from a location and a schema that two requests disagreed on. - Result> LoadFormatTable( - const Identifier& identifier) const override; - private: RestCatalog(std::shared_ptr api, const std::shared_ptr& fs, const std::string& warehouse, bool data_token_enabled, bool fs_explicitly_supplied, diff --git a/src/paimon/rest/rest_catalog_test.cpp b/src/paimon/rest/rest_catalog_test.cpp index 4b878250d..48d5bf19c 100644 --- a/src/paimon/rest/rest_catalog_test.cpp +++ b/src/paimon/rest/rest_catalog_test.cpp @@ -1146,8 +1146,10 @@ TEST_F(RestCatalogTest, CreateRefusesAFormatTableThisClientCouldNotOpen) { // And a schema this library can open is created and opens. ASSERT_OK(create("fine", good, {"dt"}, format_table)); - ASSERT_OK_AND_ASSIGN(std::shared_ptr table, - catalog->GetFormatTable(Identifier("db1", "fine"))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
loaded, + catalog->GetTable(Identifier("db1", "fine"))); + std::shared_ptr table = std::dynamic_pointer_cast(loaded); + ASSERT_NE(nullptr, table); ASSERT_EQ(FormatTable::Format::PARQUET, table->GetFormat()); } @@ -1169,7 +1171,7 @@ TEST_F(RestCatalogTest, TableResponseWithoutAPathIsRejected) { // would be checked against an empty one has no boundary at all, so the response is refused // rather than turned into a table. What the response said is not repeated back: a body may // carry credentials, so a failure to read one names the request and nothing else. - ASSERT_NOK_WITH_MSG(catalog->GetFormatTable(Identifier("db1", "no_path")), + ASSERT_NOK_WITH_MSG(catalog->GetTable(Identifier("db1", "no_path")), "failed to deserialize the response"); ASSERT_NOK_WITH_MSG(catalog->GetTableLocation(Identifier("db1", "no_path")), "failed to deserialize the response"); @@ -1192,8 +1194,10 @@ TEST_F(RestCatalogTest, FormatTableIsLoadedInOneRequest) { state_->get_table_requests = 0; } - ASSERT_OK_AND_ASSIGN(std::shared_ptr table, - catalog->GetFormatTable(Identifier("db1", "fmt"))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
loaded, + catalog->GetTable(Identifier("db1", "fmt"))); + std::shared_ptr table = std::dynamic_pointer_cast(loaded); + ASSERT_NE(nullptr, table); ASSERT_EQ("wh1/db1.db/fmt", table->Location()); ASSERT_EQ(FormatTable::Format::PARQUET, table->GetFormat()); ASSERT_EQ((std::vector{"dt"}), table->PartitionKeys()); @@ -1208,18 +1212,11 @@ TEST_F(RestCatalogTest, FormatTableIsLoadedInOneRequest) { // directory named "schema" there is a partition value, not metadata to skip. ASSERT_FALSE(table->LocationCarriesPaimonMetadata()); - ASSERT_NOK_WITH_MSG(catalog->GetTable(Identifier("db1", "fmt")), "Cannot open format table"); - - // A managed table stays out of this path, and so does a system table. + // A managed table comes back through the same call, as the managed table it is. ASSERT_OK(CreateSampleTable(catalog.get(), Identifier("db1", "t1"))); - ASSERT_NOK_WITH_MSG(catalog->GetFormatTable(Identifier("db1", "t1")), "is not a format table"); - ASSERT_NOK_WITH_MSG(catalog->GetFormatTable(Identifier("db1", "t1$snapshots")), - "for system table"); - // The database is checked before the table name is parsed, so a malformed name under "sys" - // is refused as a system table rather than surfacing a parse error. - ASSERT_NOK_WITH_MSG(catalog->GetFormatTable(Identifier("sys", "all_table_options")), - "for system table"); - ASSERT_NOK_WITH_MSG(catalog->GetFormatTable(Identifier("sys", "a$b$c$d")), "for system table"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
managed, + catalog->GetTable(Identifier("db1", "t1"))); + ASSERT_EQ(nullptr, std::dynamic_pointer_cast(managed)); } TEST_F(RestCatalogTest, FormatTableWithAnUnusableSchemaIsRejectedOnLoad) { @@ -1245,24 +1242,23 @@ TEST_F(RestCatalogTest, FormatTableWithAnUnusableSchemaIsRejectedOnLoad) { // A format table format with no reader here. seed("csv_table", R"({"type": "format-table", "file.format": "csv"})", R"(["dt"])", default_fields); - ASSERT_NOK_WITH_MSG(catalog->GetFormatTable(Identifier("db1", "csv_table")), + ASSERT_NOK_WITH_MSG(catalog->GetTable(Identifier("db1", "csv_table")), "not supported by paimon-cpp yet"); // A row count no file could ever reach. seed("bad_rows", R"({"type": "format-table", "file.format": "parquet", "target-file-row-num": "0"})", R"(["dt"])", default_fields); - ASSERT_NOK_WITH_MSG(catalog->GetFormatTable(Identifier("db1", "bad_rows")), - "should be at least 1"); + ASSERT_NOK_WITH_MSG(catalog->GetTable(Identifier("db1", "bad_rows")), "should be at least 1"); // The failure names the table, which is all a caller holding several has to go on. - ASSERT_NOK_WITH_MSG(catalog->GetFormatTable(Identifier("db1", "bad_rows")), "db1.bad_rows"); + ASSERT_NOK_WITH_MSG(catalog->GetTable(Identifier("db1", "bad_rows")), "db1.bad_rows"); // The structural rules run here too, not only the format-table ones: a partition key that is // not a field of the schema is a table nothing could ever read. seed("bad_partition_key", R"({"type": "format-table", "file.format": "parquet"})", R"(["nosuchfield"])", default_fields); - ASSERT_NOK_WITH_MSG(catalog->GetFormatTable(Identifier("db1", "bad_partition_key")), + ASSERT_NOK_WITH_MSG(catalog->GetTable(Identifier("db1", "bad_partition_key")), "should include all partition fields"); // Two schemas this library cannot open, refused when the table is opened rather than when a @@ -1272,12 +1268,12 @@ TEST_F(RestCatalogTest, FormatTableWithAnUnusableSchemaIsRejectedOnLoad) { R"([{"id": 0, "name": "f0", "type": "INT NOT NULL"},)" // Not a raw string: `TIMESTAMP(6)"` holds the sequence that would end one. " {\"id\": 1, \"name\": \"dt\", \"type\": \"TIMESTAMP(6)\"}]"); - ASSERT_NOK_WITH_MSG(catalog->GetFormatTable(Identifier("db1", "timestamp_partition")), + ASSERT_NOK_WITH_MSG(catalog->GetTable(Identifier("db1", "timestamp_partition")), "cannot be TIMESTAMP/DECIMAL/BLOB"); seed("only_partitions", R"({"type": "format-table", "file.format": "parquet"})", R"(["dt"])", R"([{"id": 0, "name": "dt", "type": "STRING"}])"); - ASSERT_NOK_WITH_MSG(catalog->GetFormatTable(Identifier("db1", "only_partitions")), + ASSERT_NOK_WITH_MSG(catalog->GetTable(Identifier("db1", "only_partitions")), "every one of its columns"); // An option this library does not honour keeps its own status code, so a caller can still @@ -1286,8 +1282,7 @@ TEST_F(RestCatalogTest, FormatTableWithAnUnusableSchemaIsRejectedOnLoad) { R"({"type": "format-table", "file.format": "parquet",)" R"( "metastore.partitioned-table": "true"})", R"(["dt"])", default_fields); - Status not_implemented = - catalog->GetFormatTable(Identifier("db1", "catalog_partitions")).status(); + Status not_implemented = catalog->GetTable(Identifier("db1", "catalog_partitions")).status(); ASSERT_TRUE(not_implemented.IsNotImplemented()) << not_implemented.ToString(); } diff --git a/src/paimon/testing/mock/mock_catalog.h b/src/paimon/testing/mock/mock_catalog.h index 67449abf2..c4a200239 100644 --- a/src/paimon/testing/mock/mock_catalog.h +++ b/src/paimon/testing/mock/mock_catalog.h @@ -34,6 +34,7 @@ #include "paimon/core/catalog/version_managed_catalog.h" #include "paimon/core/partition/partition_statistics.h" #include "paimon/core/snapshot.h" +#include "paimon/core/table/file_store_table.h" #include "paimon/result.h" #include "paimon/schema/schema.h" #include "paimon/status.h" @@ -98,7 +99,7 @@ class MockVersionManagedCatalog : public Catalog, public VersionManagedCatalog { // Serves the per-table file system when one was set, matching `GetTableFileSystem` // without recording the request, so a test can still tell the two apart. No location: // this mock's `GetTableLocation` is the unsupported stub. - return std::make_shared
( + return std::make_shared( table_file_system_ != nullptr ? table_file_system_ : file_system_, /*root_path=*/"", table_schema_, identifier, table_uuid_); } From 3fbf1bbe1c3cc40523414e30e8b87465eb4f0953 Mon Sep 17 00:00:00 2001 From: "yonghao.fyh" Date: Tue, 20 Oct 2026 16:18:14 +0800 Subject: [PATCH 02/17] fix --- include/paimon/catalog/catalog.h | 6 +- include/paimon/catalog/table.h | 14 ++-- include/paimon/table/format/format_table.h | 8 ++ .../core/catalog/file_system_catalog.cpp | 67 ++++++++------- src/paimon/core/catalog/file_system_catalog.h | 5 ++ .../core/catalog/file_system_catalog_test.cpp | 83 ++++++++++++++++++- src/paimon/core/table/file_store_table.h | 8 ++ .../table/system/audit_log_system_table.cpp | 9 +- .../table/system/audit_log_system_table.h | 6 +- .../core/table/system/binlog_system_table.cpp | 3 +- .../core/table/system/binlog_system_table.h | 3 +- .../table/system/global_system_tables.cpp | 16 ++-- .../table/system/in_memory_system_table.cpp | 11 ++- .../table/system/in_memory_system_table.h | 9 +- .../table/system/metadata_system_tables.cpp | 19 +++-- .../table/system/metadata_system_tables.h | 3 +- .../system/read_optimized_system_table.cpp | 12 ++- .../system/read_optimized_system_table.h | 10 +-- src/paimon/core/table/system/system_table.cpp | 41 +++++++-- src/paimon/core/table/system/system_table.h | 46 +++++++--- .../core/table/system/system_table_test.cpp | 2 +- src/paimon/rest/rest_catalog.cpp | 65 ++++++++------- src/paimon/rest/rest_catalog.h | 7 ++ src/paimon/rest/rest_catalog_test.cpp | 9 +- test/inte/read_inte_test.cpp | 8 +- 25 files changed, 330 insertions(+), 140 deletions(-) diff --git a/include/paimon/catalog/catalog.h b/include/paimon/catalog/catalog.h index d501bc7e3..0bb578383 100644 --- a/include/paimon/catalog/catalog.h +++ b/include/paimon/catalog/catalog.h @@ -149,8 +149,10 @@ class PAIMON_EXPORT Catalog { /// /// The table's `type` option decides what is handed back: a `format-table` comes back as /// the `FormatTable` it is - a directory of data files, with no snapshot and no manifest - - /// and every other implemented type as the managed table it is. Both go through this one - /// `Table` interface: `NewScan()`, `NewRead()`, `NewWrite()` and `NewCommit()` serve either. + /// and every other implemented type as the managed table it is. A `T$system` identifier + /// names a read-only view instead, which comes back as the `SystemTable` it is. All go + /// through this one `Table` interface: `NewScan()` and `NewRead()` serve any of them, + /// `NewWrite()` and `NewCommit()` the writable ones. /// /// @param identifier Identifier of the table to get. /// @return A result containing the table, or an error status. diff --git a/include/paimon/catalog/table.h b/include/paimon/catalog/table.h index d18bd07b8..17b3b3d21 100644 --- a/include/paimon/catalog/table.h +++ b/include/paimon/catalog/table.h @@ -66,15 +66,13 @@ class PAIMON_EXPORT Table { return root_path_; } - /// A name to identify this table. - std::string Name() const { - return identifier_.GetObjectName(); - } + /// A name to identify this table, which each kind of table answers for itself: a managed or + /// format table is named by its identifier, a system table by its own kind. + virtual std::string Name() const = 0; - /// Full name of the table, default is database.tableName. - std::string FullName() const { - return identifier_.GetFullName(); - } + /// Full name of the table, which each kind of table answers for itself. A managed or format + /// table answers database.tableName, a system table the same name as `Name()`. + virtual std::string FullName() const = 0; /// Returns the branch the table was opened for, which its identifier carries; `main` /// when it names none or a spelling of it, which its managers are bound to. A diff --git a/include/paimon/table/format/format_table.h b/include/paimon/table/format/format_table.h index d2db6aa4a..d15a8f6ce 100644 --- a/include/paimon/table/format/format_table.h +++ b/include/paimon/table/format/format_table.h @@ -123,6 +123,14 @@ class PAIMON_EXPORT FormatTable : public Table, public std::enable_shared_from_t ~FormatTable(); + std::string Name() const override { + return identifier_.GetObjectName(); + } + + std::string FullName() const override { + return identifier_.GetFullName(); + } + /// Directory the data files live in. const std::string& Location() const { return GetRootPath(); diff --git a/src/paimon/core/catalog/file_system_catalog.cpp b/src/paimon/core/catalog/file_system_catalog.cpp index 14293021e..46131f729 100644 --- a/src/paimon/core/catalog/file_system_catalog.cpp +++ b/src/paimon/core/catalog/file_system_catalog.cpp @@ -294,40 +294,16 @@ Result> FileSystemCatalog::LoadTableSchema( context.catalog_options = catalog_options_; PAIMON_ASSIGN_OR_RAISE(std::shared_ptr system_table, GlobalSystemTableLoader::Load(identifier.GetObjectName(), context)); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr arrow_schema, - system_table->ArrowSchema()); - return std::make_shared(std::move(arrow_schema)); + return system_table->LatestSchema(); } // The system table suffix is dropped when the data table identifier is rebuilt below, so the // identifier is validated as a whole here. PAIMON_RETURN_NOT_OK(CatalogUtils::CheckValidTableName(identifier)); PAIMON_ASSIGN_OR_RAISE(bool is_system_table, identifier.IsSystemTable()); if (is_system_table) { - PAIMON_ASSIGN_OR_RAISE(std::optional system_table_name, - identifier.GetSystemTableName()); - if (!system_table_name || !SystemTableLoader::IsSupported(system_table_name.value())) { - return Status::NotExist(fmt::format("{} not exist", identifier.ToString())); - } - PAIMON_ASSIGN_OR_RAISE(std::optional branch, identifier.GetBranchName()); - PAIMON_ASSIGN_OR_RAISE( - Identifier data_identifier, - CatalogUtils::BranchIdentifier(identifier, branch.value_or(std::string()))); - PAIMON_ASSIGN_OR_RAISE(std::optional> latest_schema, - TableSchemaExists(data_identifier)); - if (!latest_schema) { - return Status::NotExist(fmt::format("{} not exist", data_identifier.ToString())); - } - std::map dynamic_options; - if (branch) { - dynamic_options[Options::BRANCH] = branch.value(); - } - PAIMON_ASSIGN_OR_RAISE(std::string table_path, GetTableLocation(data_identifier)); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr system_table, - SystemTableLoader::Load(system_table_name.value(), fs_, table_path, - latest_schema.value(), dynamic_options)); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr arrow_schema, - system_table->ArrowSchema()); - return std::make_shared(std::move(arrow_schema)); + LoadSystemTable(identifier)); + return system_table->LatestSchema(); } PAIMON_ASSIGN_OR_RAISE(std::optional> latest_schema, TableSchemaExists(identifier)); @@ -340,13 +316,9 @@ Result> FileSystemCatalog::LoadTableSchema( Result> FileSystemCatalog::GetTable(const Identifier& identifier) const { PAIMON_ASSIGN_OR_RAISE(bool is_system_table, identifier.IsSystemTable()); if (is_system_table) { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, LoadTableSchema(identifier)); - // A system table reads the files of the data table it belongs to, so its root path - // is that table's. - PAIMON_ASSIGN_OR_RAISE(std::string data_table_name, identifier.GetDataTableName()); - Identifier data_identifier(identifier.GetDatabaseName(), data_table_name); - PAIMON_ASSIGN_OR_RAISE(std::string table_path, GetTableLocation(data_identifier)); - return std::make_shared(fs_, table_path, schema, identifier); + // A system table reads the files of the data table it belongs to, through the file + // system of that table's catalog, and the schema it exposes is its own read-only one. + return LoadSystemTable(identifier); } // The schema is read on the table's own branch: `root/branch/branch-` below the // table path, which `main` resolves to itself. `GetBranchNameOrDefault` folds any @@ -379,6 +351,33 @@ Result> FileSystemCatalog::GetTable(const Identifier& ide fs_, table_path, checked_pointer_cast(*latest_schema), identifier); } +Result> FileSystemCatalog::LoadSystemTable( + const Identifier& identifier) const { + PAIMON_ASSIGN_OR_RAISE(std::optional system_table_name, + identifier.GetSystemTableName()); + if (!system_table_name || !SystemTableLoader::IsSupported(system_table_name.value())) { + return Status::NotExist(fmt::format("{} not exist", identifier.ToString())); + } + PAIMON_ASSIGN_OR_RAISE(std::optional branch, identifier.GetBranchName()); + PAIMON_ASSIGN_OR_RAISE( + Identifier data_identifier, + CatalogUtils::BranchIdentifier(identifier, branch.value_or(std::string()))); + PAIMON_ASSIGN_OR_RAISE(std::optional> latest_schema, + TableSchemaExists(data_identifier)); + if (!latest_schema) { + return Status::NotExist(fmt::format("{} not exist", data_identifier.ToString())); + } + std::map dynamic_options; + if (branch) { + dynamic_options[Options::BRANCH] = branch.value(); + } + // A system table reads the files of the data table it belongs to, so its root path is + // that table's. + PAIMON_ASSIGN_OR_RAISE(std::string table_path, GetTableLocation(data_identifier)); + return SystemTableLoader::Load(system_table_name.value(), fs_, table_path, + latest_schema.value(), dynamic_options); +} + Status FileSystemCatalog::DropDatabase(const std::string& name, bool ignore_if_not_exists, bool cascade) { PAIMON_RETURN_NOT_OK(CatalogUtils::CheckNotSystemDatabase(name, "dropDatabase")); diff --git a/src/paimon/core/catalog/file_system_catalog.h b/src/paimon/core/catalog/file_system_catalog.h index fd4360755..203e1b92e 100644 --- a/src/paimon/core/catalog/file_system_catalog.h +++ b/src/paimon/core/catalog/file_system_catalog.h @@ -37,6 +37,7 @@ class TableSchema; class FileSystem; class Identifier; class Logger; +class SystemTable; class FileSystemCatalog : public Catalog { public: @@ -85,6 +86,10 @@ class FileSystemCatalog : public Catalog { Result>> TableSchemaExists( const Identifier& identifier) const; + /// Loads the system table a `T$system` identifier names, on the branch the identifier + /// carries. The base table it belongs to must exist and carry a schema. + Result> LoadSystemTable(const Identifier& identifier) const; + Status CreateDatabaseImpl(const std::string& db_name, const std::map& options); Result TableExistsInFileSystem(const std::string& table_path) const; diff --git a/src/paimon/core/catalog/file_system_catalog_test.cpp b/src/paimon/core/catalog/file_system_catalog_test.cpp index cfe361bae..f31c0391d 100644 --- a/src/paimon/core/catalog/file_system_catalog_test.cpp +++ b/src/paimon/core/catalog/file_system_catalog_test.cpp @@ -34,14 +34,21 @@ #include "paimon/core/schema/schema_manager.h" #include "paimon/core/schema/table_schema.h" #include "paimon/core/table/system/global_system_tables.h" +#include "paimon/core/table/system/system_table.h" #include "paimon/core/table/system/system_table_schema.h" #include "paimon/defs.h" #include "paimon/file_store_commit.h" +#include "paimon/file_store_write.h" #include "paimon/fs/file_system.h" #include "paimon/fs/file_system_factory.h" +#include "paimon/read_context.h" +#include "paimon/scan_context.h" #include "paimon/snapshot/snapshot_info.h" #include "paimon/table/format/format_table.h" +#include "paimon/table/source/table_read.h" +#include "paimon/table/source/table_scan.h" #include "paimon/testing/utils/testharness.h" +#include "paimon/write_context.h" namespace paimon::test { @@ -223,10 +230,15 @@ TEST(FileSystemCatalogTest, TestOptionsSystemTableCatalog) { ASSERT_FALSE(loaded_schema->field(1)->nullable()); ASSERT_OK_AND_ASSIGN(auto system_table, catalog.GetTable(options_identifier)); - ASSERT_EQ(system_table->Name(), "tbl1$options"); - // A system table reads the files of the data table it belongs to, so the table opened - // for it carries that table's root path. + // A system table comes back as the `SystemTable` it is: it answers for itself with its own + // read-only schema and the root path of the data table it reads. + ASSERT_TRUE(std::dynamic_pointer_cast(system_table) != nullptr); + ASSERT_TRUE(std::dynamic_pointer_cast(system_table) == nullptr); + ASSERT_EQ("options", system_table->Name()); + ASSERT_EQ("options", system_table->FullName()); ASSERT_EQ(system_table->GetRootPath(), table_path); + ASSERT_EQ((std::vector{"key", "value"}), + system_table->LatestSchema()->FieldNames()); ASSERT_NOK_WITH_MSG(catalog.LoadTableSchema(Identifier("db1", "tbl1$unknown")), "not exist"); ASSERT_NOK_WITH_MSG(catalog.LoadTableSchema(Identifier("db1", "missing$options")), "not exist"); @@ -242,6 +254,71 @@ TEST(FileSystemCatalogTest, TestOptionsSystemTableCatalog) { "Cannot 'renameTable' for system table"); } +// A `T$system` identifier opens the read-only view it names, which serves scans and reads of +// the data table it belongs to through the same `Table` interface, and refuses the operations +// that would modify the data table. +TEST(FileSystemCatalogTest, TestGetTableServesASystemTable) { + std::map options; + options[Options::FILE_SYSTEM] = "local"; + options[Options::FILE_FORMAT] = "orc"; + ASSERT_OK_AND_ASSIGN(auto core_options, CoreOptions::FromMap(options)); + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + FileSystemCatalog catalog(core_options.GetFileSystem(), dir->Str(), options); + ASSERT_OK(catalog.CreateDatabase("db1", options, /*ignore_if_exists=*/true)); + + auto typed_schema = arrow::schema({arrow::field("f0", arrow::int32(), /*nullable=*/false), + arrow::field("dt", arrow::utf8())}); + ::ArrowSchema schema; + ASSERT_TRUE(arrow::ExportSchema(*typed_schema, &schema).ok()); + ASSERT_OK(catalog.CreateTable(Identifier("db1", "tbl1"), &schema, /*partition_keys=*/{"dt"}, + /*primary_keys=*/{}, options, /*ignore_if_exists=*/false)); + ArrowSchemaRelease(&schema); + + // A managed table with no `type` option still opens, as the managed table it is. + ASSERT_OK_AND_ASSIGN(auto managed, catalog.GetTable(Identifier("db1", "tbl1"))); + ASSERT_TRUE(std::dynamic_pointer_cast(managed) == nullptr); + + ASSERT_OK_AND_ASSIGN(auto system_table, catalog.GetTable(Identifier("db1", "tbl1$options"))); + ASSERT_TRUE(std::dynamic_pointer_cast(system_table) != nullptr); + ASSERT_EQ("options", system_table->Name()); + + // The scan and the read work from a context of the caller's, like every other table. + ScanContextBuilder scan_builder(system_table->GetRootPath()); + scan_builder.SetOptions(options); + ASSERT_OK_AND_ASSIGN(auto scan_context, scan_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan, system_table->NewScan(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto plan, scan->CreatePlan()); + + ReadContextBuilder read_builder(system_table->GetRootPath()); + read_builder.SetOptions(options); + ASSERT_OK_AND_ASSIGN(auto read_context, read_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto read, system_table->NewRead(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto reader, read->CreateReader(plan->Splits())); + ASSERT_OK_AND_ASSIGN(auto table_result, reader->NextBatch()); + + // The options the table was created with come back, one row per key. + auto& [c_array, c_schema] = table_result; + std::shared_ptr batch; + auto maybe_batch = arrow::ImportRecordBatch(c_array.get(), c_schema.get()); + ASSERT_TRUE(maybe_batch.ok()) << maybe_batch.status().ToString(); + batch = std::move(maybe_batch).ValueUnsafe(); + ASSERT_GE(batch->num_rows(), 2); + + // The operations that would modify the data table underneath are refused by name. + WriteContextBuilder write_builder(system_table->GetRootPath(), "commit_user"); + write_builder.SetOptions(options); + ASSERT_OK_AND_ASSIGN(auto write_context, write_builder.Finish()); + ASSERT_NOK_WITH_MSG(system_table->NewWrite(std::move(write_context)), + "options system table is read-only: writing is not supported"); + + CommitContextBuilder commit_builder(system_table->GetRootPath(), "commit_user"); + commit_builder.SetOptions(options); + ASSERT_OK_AND_ASSIGN(auto commit_context, commit_builder.Finish()); + ASSERT_NOK_WITH_MSG(system_table->NewCommit(std::move(commit_context)), + "options system table is read-only: committing for it is not supported"); +} + TEST(FileSystemCatalogTest, TestBranchTableLoadsBranchSchema) { std::map options; options[Options::FILE_SYSTEM] = "local"; diff --git a/src/paimon/core/table/file_store_table.h b/src/paimon/core/table/file_store_table.h index 409fefca9..f02b4e528 100644 --- a/src/paimon/core/table/file_store_table.h +++ b/src/paimon/core/table/file_store_table.h @@ -51,6 +51,14 @@ class FileStoreTable : public Table { const std::string& uuid = "") : Table(file_system, root_path, schema, identifier, uuid) {} + std::string Name() const override { + return identifier_.GetObjectName(); + } + + std::string FullName() const override { + return identifier_.GetFullName(); + } + Result> NewScan(std::unique_ptr context) const override; Result> NewRead(std::unique_ptr context) const override; diff --git a/src/paimon/core/table/system/audit_log_system_table.cpp b/src/paimon/core/table/system/audit_log_system_table.cpp index 8de9742e1..3d3f7b3e5 100644 --- a/src/paimon/core/table/system/audit_log_system_table.cpp +++ b/src/paimon/core/table/system/audit_log_system_table.cpp @@ -358,7 +358,8 @@ std::unique_ptr CreateChangelogBatchReader( AuditLogSystemTable::AuditLogSystemTable(std::shared_ptr fs, std::string table_path, std::shared_ptr table_schema, std::map options) - : fs_(std::move(fs)), + : SystemTable(fs, table_path), + fs_(std::move(fs)), table_path_(std::move(table_path)), table_schema_(std::move(table_schema)), options_(std::move(options)) {} @@ -383,7 +384,8 @@ Result> AuditLogSystemTable::ArrowSchema() const } Result> AuditLogSystemTable::NewScan( - const std::shared_ptr& context) const { + std::unique_ptr ctx) const { + const std::shared_ptr context = std::move(ctx); if (context->GetScanFilters() && context->GetScanFilters()->GetPredicate()) { return Status::NotImplemented("audit_log system table predicate pushdown is not supported"); } @@ -411,7 +413,8 @@ Result> AuditLogSystemTable::NewScan( } Result> AuditLogSystemTable::NewRead( - const std::shared_ptr& context) const { + std::unique_ptr ctx) const { + const std::shared_ptr context = std::move(ctx); return NewChangelogRead(context, std::make_shared()); } diff --git a/src/paimon/core/table/system/audit_log_system_table.h b/src/paimon/core/table/system/audit_log_system_table.h index 780f5fa72..7395c9797 100644 --- a/src/paimon/core/table/system/audit_log_system_table.h +++ b/src/paimon/core/table/system/audit_log_system_table.h @@ -60,10 +60,8 @@ class AuditLogSystemTable : public SystemTable { std::string Name() const override; Result> ArrowSchema() const override; - Result> NewScan( - const std::shared_ptr& context) const override; - Result> NewRead( - const std::shared_ptr& context) const override; + Result> NewScan(std::unique_ptr context) const override; + Result> NewRead(std::unique_ptr context) const override; protected: Result> NewChangelogRead( diff --git a/src/paimon/core/table/system/binlog_system_table.cpp b/src/paimon/core/table/system/binlog_system_table.cpp index 88541194d..8a821d45f 100644 --- a/src/paimon/core/table/system/binlog_system_table.cpp +++ b/src/paimon/core/table/system/binlog_system_table.cpp @@ -98,7 +98,8 @@ Result> BinlogSystemTable::ArrowSchema() const { } Result> BinlogSystemTable::NewRead( - const std::shared_ptr& context) const { + std::unique_ptr ctx) const { + const std::shared_ptr context = std::move(ctx); return NewChangelogRead(context, CreateBinlogBatchConverter()); } diff --git a/src/paimon/core/table/system/binlog_system_table.h b/src/paimon/core/table/system/binlog_system_table.h index 3b4f45bd0..3a939fcef 100644 --- a/src/paimon/core/table/system/binlog_system_table.h +++ b/src/paimon/core/table/system/binlog_system_table.h @@ -42,8 +42,7 @@ class BinlogSystemTable : public AuditLogSystemTable { std::string Name() const override; Result> ArrowSchema() const override; - Result> NewRead( - const std::shared_ptr& context) const override; + Result> NewRead(std::unique_ptr context) const override; }; } // namespace paimon diff --git a/src/paimon/core/table/system/global_system_tables.cpp b/src/paimon/core/table/system/global_system_tables.cpp index 647da76c3..a66362b39 100644 --- a/src/paimon/core/table/system/global_system_tables.cpp +++ b/src/paimon/core/table/system/global_system_tables.cpp @@ -300,7 +300,13 @@ Result> GlobalSystemTableLoader::Load( return Status::NotImplemented("global system table requires catalog context: ", table_name); } - return entry.factory(context); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr system_table, + entry.factory(context)); + // The table is handed out through the `Table` interface, which serves the schema + // the base holds, so its own read-only schema is adopted before it leaves the + // loader. + PAIMON_RETURN_NOT_OK(system_table->AdoptOwnSchema()); + return system_table; } } return Status::NotImplemented("unsupported global system table: ", table_name); @@ -324,7 +330,7 @@ Result> GlobalSystemTableLoader::GetSupportedTableNames // ============================================================================= CatalogOptionsSystemTable::CatalogOptionsSystemTable(GlobalSystemTableContext context) - : InMemorySystemTable("sys/catalog_options"), context_(std::move(context)) {} + : InMemorySystemTable(context.fs, "sys/catalog_options"), context_(std::move(context)) {} std::string CatalogOptionsSystemTable::Name() const { return kName; @@ -360,7 +366,7 @@ Result> CatalogOptionsSystemTable::BuildRows() const { // ============================================================================= AllTableOptionsSystemTable::AllTableOptionsSystemTable(GlobalSystemTableContext context) - : InMemorySystemTable("sys/all_table_options"), context_(std::move(context)) {} + : InMemorySystemTable(context.fs, "sys/all_table_options"), context_(std::move(context)) {} std::string AllTableOptionsSystemTable::Name() const { return kName; @@ -399,7 +405,7 @@ Result> AllTableOptionsSystemTable::BuildRows() const { // ============================================================================= TablesSystemTable::TablesSystemTable(GlobalSystemTableContext context) - : InMemorySystemTable("sys/tables"), context_(std::move(context)) {} + : InMemorySystemTable(context.fs, "sys/tables"), context_(std::move(context)) {} std::string TablesSystemTable::Name() const { return kName; @@ -471,7 +477,7 @@ Result> TablesSystemTable::BuildRows() const { // ============================================================================= PartitionsSystemTable::PartitionsSystemTable(GlobalSystemTableContext context) - : InMemorySystemTable("sys/partitions"), context_(std::move(context)) {} + : InMemorySystemTable(context.fs, "sys/partitions"), context_(std::move(context)) {} std::string PartitionsSystemTable::Name() const { return kName; diff --git a/src/paimon/core/table/system/in_memory_system_table.cpp b/src/paimon/core/table/system/in_memory_system_table.cpp index 8d6111d79..773f621b8 100644 --- a/src/paimon/core/table/system/in_memory_system_table.cpp +++ b/src/paimon/core/table/system/in_memory_system_table.cpp @@ -31,6 +31,7 @@ #include "paimon/core/table/system/system_table_scan.h" #include "paimon/memory/memory_pool.h" #include "paimon/read_context.h" +#include "paimon/scan_context.h" #include "paimon/status.h" #include "paimon/table/source/table_read.h" @@ -104,16 +105,18 @@ class InMemorySystemTableRead : public TableRead { } // namespace -InMemorySystemTable::InMemorySystemTable(std::string table_path) - : table_path_(std::move(table_path)) {} +InMemorySystemTable::InMemorySystemTable(const std::shared_ptr& fs, + std::string table_path) + : SystemTable(fs, table_path), table_path_(std::move(table_path)) {} Result> InMemorySystemTable::NewScan( - const std::shared_ptr& /*context*/) const { + std::unique_ptr /*context*/) const { return std::make_unique(table_path_); } Result> InMemorySystemTable::NewRead( - const std::shared_ptr& context) const { + std::unique_ptr ctx) const { + const std::shared_ptr context = std::move(ctx); return std::make_unique( checked_pointer_cast(shared_from_this()), context->GetMemoryPool()); diff --git a/src/paimon/core/table/system/in_memory_system_table.h b/src/paimon/core/table/system/in_memory_system_table.h index f9fcebdb3..90cd46d84 100644 --- a/src/paimon/core/table/system/in_memory_system_table.h +++ b/src/paimon/core/table/system/in_memory_system_table.h @@ -28,18 +28,19 @@ namespace paimon { +class FileSystem; + /// Base class for system tables whose result can be materialized as a single in-memory /// RecordBatch. /// /// It provides the common singleton split scan and one-shot batch reader. class InMemorySystemTable : public SystemTable { public: - explicit InMemorySystemTable(std::string table_path); + InMemorySystemTable(const std::shared_ptr& fs, std::string table_path); Result> NewScan( - const std::shared_ptr& context) const override; - Result> NewRead( - const std::shared_ptr& context) const override; + std::unique_ptr /*context*/) const override; + Result> NewRead(std::unique_ptr context) const override; virtual Result> BuildRows() const = 0; protected: diff --git a/src/paimon/core/table/system/metadata_system_tables.cpp b/src/paimon/core/table/system/metadata_system_tables.cpp index 5279e6871..e9e00ea42 100644 --- a/src/paimon/core/table/system/metadata_system_tables.cpp +++ b/src/paimon/core/table/system/metadata_system_tables.cpp @@ -504,9 +504,10 @@ Result> WriteColsValue( } // namespace -OptionsSystemTable::OptionsSystemTable(std::string table_path, +OptionsSystemTable::OptionsSystemTable(const std::shared_ptr& fs, + std::string table_path, std::shared_ptr table_schema) - : InMemorySystemTable(std::move(table_path)), table_schema_(std::move(table_schema)) {} + : InMemorySystemTable(fs, table_path), table_schema_(std::move(table_schema)) {} std::string OptionsSystemTable::Name() const { return kName; @@ -532,7 +533,7 @@ Result> OptionsSystemTable::BuildRows() const { SnapshotsSystemTable::SnapshotsSystemTable(std::shared_ptr fs, std::string table_path, std::string branch) - : InMemorySystemTable(table_path), + : InMemorySystemTable(fs, table_path), context_(CreateMetadataContext(std::move(fs), std::move(table_path), std::move(branch))) {} std::string SnapshotsSystemTable::Name() const { @@ -594,7 +595,7 @@ Result> SnapshotsSystemTable::BuildRows() const { SchemasSystemTable::SchemasSystemTable(std::shared_ptr fs, std::string table_path, std::string branch) - : InMemorySystemTable(table_path), + : InMemorySystemTable(fs, table_path), context_(CreateMetadataContext(std::move(fs), std::move(table_path), std::move(branch))) {} std::string SchemasSystemTable::Name() const { @@ -650,7 +651,7 @@ Result> SchemasSystemTable::BuildRows() const { TagsSystemTable::TagsSystemTable(std::shared_ptr fs, std::string table_path, std::string branch) - : InMemorySystemTable(table_path), + : InMemorySystemTable(fs, table_path), context_(CreateMetadataContext(std::move(fs), std::move(table_path), std::move(branch))) {} std::string TagsSystemTable::Name() const { @@ -700,7 +701,7 @@ Result> TagsSystemTable::BuildRows() const { BranchesSystemTable::BranchesSystemTable(std::shared_ptr fs, std::string table_path, std::string branch) - : InMemorySystemTable(table_path), + : InMemorySystemTable(fs, table_path), context_(CreateMetadataContext(std::move(fs), std::move(table_path), std::move(branch))) {} std::string BranchesSystemTable::Name() const { @@ -739,7 +740,7 @@ Result> BranchesSystemTable::BuildRows() const { ConsumersSystemTable::ConsumersSystemTable(std::shared_ptr fs, std::string table_path, std::string branch) - : InMemorySystemTable(table_path), + : InMemorySystemTable(fs, table_path), context_(CreateMetadataContext(std::move(fs), std::move(table_path), std::move(branch))) {} std::string ConsumersSystemTable::Name() const { @@ -774,7 +775,7 @@ ManifestsSystemTable::ManifestsSystemTable(std::shared_ptr fs, std:: std::string branch, std::shared_ptr table_schema, std::map options) - : InMemorySystemTable(table_path), + : InMemorySystemTable(fs, table_path), context_(CreateMetadataContext(std::move(fs), std::move(table_path), std::move(branch), std::move(table_schema), std::move(options))) {} @@ -843,7 +844,7 @@ Result> ManifestsSystemTable::BuildRows() const { FilesSystemTable::FilesSystemTable(std::shared_ptr fs, std::string table_path, std::string branch, std::shared_ptr table_schema, std::map options) - : InMemorySystemTable(table_path), + : InMemorySystemTable(fs, table_path), context_(CreateMetadataContext(std::move(fs), std::move(table_path), std::move(branch), std::move(table_schema), std::move(options))) {} diff --git a/src/paimon/core/table/system/metadata_system_tables.h b/src/paimon/core/table/system/metadata_system_tables.h index e3961e476..ae2e5155b 100644 --- a/src/paimon/core/table/system/metadata_system_tables.h +++ b/src/paimon/core/table/system/metadata_system_tables.h @@ -35,7 +35,8 @@ class OptionsSystemTable : public InMemorySystemTable { public: static constexpr const char* kName = "options"; - OptionsSystemTable(std::string table_path, std::shared_ptr table_schema); + OptionsSystemTable(const std::shared_ptr& fs, std::string table_path, + std::shared_ptr table_schema); std::string Name() const override; Result> ArrowSchema() const override; diff --git a/src/paimon/core/table/system/read_optimized_system_table.cpp b/src/paimon/core/table/system/read_optimized_system_table.cpp index 8d5696447..172fc964c 100644 --- a/src/paimon/core/table/system/read_optimized_system_table.cpp +++ b/src/paimon/core/table/system/read_optimized_system_table.cpp @@ -35,10 +35,12 @@ namespace paimon { -ReadOptimizedSystemTable::ReadOptimizedSystemTable(std::string table_path, +ReadOptimizedSystemTable::ReadOptimizedSystemTable(const std::shared_ptr& fs, + std::string table_path, std::shared_ptr table_schema, std::map options) - : table_path_(std::move(table_path)), + : SystemTable(fs, table_path), + table_path_(std::move(table_path)), table_schema_(std::move(table_schema)), options_(std::move(options)) {} @@ -57,7 +59,8 @@ std::map ReadOptimizedSystemTable::ReadOptimizedOption } Result> ReadOptimizedSystemTable::NewScan( - const std::shared_ptr& context) const { + std::unique_ptr ctx) const { + const std::shared_ptr context = std::move(ctx); if (context->GetRealtimeContext() && !table_schema_->PrimaryKeys().empty()) { return Status::NotImplemented( "PK real-time union read does not support read-optimized scans"); @@ -133,7 +136,8 @@ Result> ReadOptimizedSystemTable::CreateDataReadCon } Result> ReadOptimizedSystemTable::NewRead( - const std::shared_ptr& context) const { + std::unique_ptr ctx) const { + const std::shared_ptr context = std::move(ctx); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr base_context, CreateDataReadContext(context)); return TableRead::Create(std::move(base_context)); diff --git a/src/paimon/core/table/system/read_optimized_system_table.h b/src/paimon/core/table/system/read_optimized_system_table.h index 3123fe03b..7f92e04e2 100644 --- a/src/paimon/core/table/system/read_optimized_system_table.h +++ b/src/paimon/core/table/system/read_optimized_system_table.h @@ -26,6 +26,7 @@ #include "paimon/core/table/system/system_table.h" namespace paimon { +class FileSystem; class TableSchema; /// System table for `T$ro`, exposing read-optimized data. @@ -33,15 +34,14 @@ class ReadOptimizedSystemTable : public SystemTable { public: static constexpr const char* kName = "ro"; - ReadOptimizedSystemTable(std::string table_path, std::shared_ptr table_schema, + ReadOptimizedSystemTable(const std::shared_ptr& fs, std::string table_path, + std::shared_ptr table_schema, std::map options); std::string Name() const override; Result> ArrowSchema() const override; - Result> NewScan( - const std::shared_ptr& context) const override; - Result> NewRead( - const std::shared_ptr& context) const override; + Result> NewScan(std::unique_ptr context) const override; + Result> NewRead(std::unique_ptr context) const override; private: /// Build the context for the data table underneath this read-optimized view. diff --git a/src/paimon/core/table/system/system_table.cpp b/src/paimon/core/table/system/system_table.cpp index 5a1a1de00..dd00b7bcd 100644 --- a/src/paimon/core/table/system/system_table.cpp +++ b/src/paimon/core/table/system/system_table.cpp @@ -28,6 +28,7 @@ #include "paimon/catalog/catalog.h" #include "paimon/catalog/identifier.h" +#include "paimon/commit_context.h" #include "paimon/common/utils/path_util.h" #include "paimon/common/utils/string_utils.h" #include "paimon/core/schema/schema_manager.h" @@ -37,8 +38,12 @@ #include "paimon/core/table/system/global_system_tables.h" #include "paimon/core/table/system/metadata_system_tables.h" #include "paimon/core/table/system/read_optimized_system_table.h" +#include "paimon/core/table/system/system_table_schema.h" #include "paimon/core/utils/branch_manager.h" +#include "paimon/file_store_commit.h" +#include "paimon/file_store_write.h" #include "paimon/status.h" +#include "paimon/write_context.h" namespace paimon { namespace { @@ -70,11 +75,11 @@ std::string LoadBranch(const std::map& options) { const std::vector& SystemTableRegistry() { static const std::vector registry = { {OptionsSystemTable::kName, - [](const std::shared_ptr& /*fs*/, const std::string& table_path, + [](const std::shared_ptr& fs, const std::string& table_path, const std::shared_ptr& table_schema, const std::map& /*dynamic_options*/) -> Result> { - return std::make_shared(table_path, table_schema); + return std::make_shared(fs, table_path, table_schema); }}, {AuditLogSystemTable::kName, [](const std::shared_ptr& fs, const std::string& table_path, @@ -93,12 +98,12 @@ const std::vector& SystemTableRegistry() { fs, table_path, table_schema, MergeOptions(table_schema, dynamic_options)); }}, {ReadOptimizedSystemTable::kName, - [](const std::shared_ptr& /*fs*/, const std::string& table_path, + [](const std::shared_ptr& fs, const std::string& table_path, const std::shared_ptr& table_schema, const std::map& dynamic_options) -> Result> { return std::make_shared( - table_path, table_schema, MergeOptions(table_schema, dynamic_options)); + fs, table_path, table_schema, MergeOptions(table_schema, dynamic_options)); }}, {SnapshotsSystemTable::kName, [](const std::shared_ptr& fs, const std::string& table_path, @@ -176,6 +181,27 @@ std::optional FindSystemTableFactory(const std::string& syst } // namespace +SystemTable::SystemTable(const std::shared_ptr& fs, const std::string& table_path) + : Table(fs, table_path, /*schema=*/nullptr, + Identifier(Identifier::kUnknownDatabase, "system_table"), /*uuid=*/"") {} + +Status SystemTable::AdoptOwnSchema() { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr arrow_schema, ArrowSchema()); + schema_ = std::make_shared(std::move(arrow_schema)); + return Status::OK(); +} + +Result> SystemTable::NewWrite( + std::unique_ptr /*context*/) const { + return Status::NotImplemented(Name(), " system table is read-only: writing is not supported"); +} + +Result> SystemTable::NewCommit( + std::unique_ptr /*context*/) const { + return Status::NotImplemented(Name(), + " system table is read-only: committing for it is not supported"); +} + bool SystemTableLoader::IsSupported(const std::string& system_table_name) { return FindSystemTableFactory(system_table_name).has_value(); } @@ -186,7 +212,12 @@ Result> SystemTableLoader::Load( const std::map& dynamic_options) { std::optional factory = FindSystemTableFactory(system_table_name); if (factory) { - return factory.value()(fs, table_path, table_schema, dynamic_options); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr system_table, + factory.value()(fs, table_path, table_schema, dynamic_options)); + // The table is handed out through the `Table` interface, which serves the schema the + // base holds, so its own read-only schema is adopted before it leaves the loader. + PAIMON_RETURN_NOT_OK(system_table->AdoptOwnSchema()); + return system_table; } return Status::NotImplemented("unsupported system table: ", system_table_name); } diff --git a/src/paimon/core/table/system/system_table.h b/src/paimon/core/table/system/system_table.h index 39e1da31b..2e4dff55d 100644 --- a/src/paimon/core/table/system/system_table.h +++ b/src/paimon/core/table/system/system_table.h @@ -25,6 +25,8 @@ #include #include "arrow/api.h" +#include "paimon/catalog/identifier.h" +#include "paimon/catalog/table.h" #include "paimon/result.h" #include "paimon/status.h" #include "paimon/type_fwd.h" @@ -33,9 +35,8 @@ namespace paimon { class FileSystem; class ReadContext; class ScanContext; -class TableScan; -class TableRead; class TableSchema; +class SystemTableSchema; /// Parsed information for a table-scoped system table path. struct SystemTablePath { @@ -49,19 +50,42 @@ struct SystemTablePath { bool is_global = false; }; -/// Base interface for table-scoped system tables such as `T$options` and `T$snapshots`. +/// Base class for table-scoped system tables such as `T$options` and `T$snapshots`. /// -/// Implementations expose a read-only schema and create their own scan/read objects. -class SystemTable : public std::enable_shared_from_this { +/// A system table is the read-only view a `T$system` identifier names: it exposes a schema of +/// its own and serves scans and reads from the data table it belongs to, through the same +/// `Table` interface as every other kind of table. `Name()` - which each kind of system table +/// answers for itself - is its full name as well: the identifier it is opened by belongs to the +/// data table underneath and would name that one. +class SystemTable : public Table, public std::enable_shared_from_this { public: - virtual ~SystemTable() = default; + std::string FullName() const override { + return Name(); + } - virtual std::string Name() const = 0; + /// The read-only schema this system table exposes, which is not the data table's schema. virtual Result> ArrowSchema() const = 0; - virtual Result> NewScan( - const std::shared_ptr& context) const = 0; - virtual Result> NewRead( - const std::shared_ptr& context) const = 0; + + /// A system table is read-only: writing it or committing for it is refused rather than + /// routed to the data table underneath. + Result> NewWrite( + std::unique_ptr context) const override; + + Result> NewCommit( + std::unique_ptr context) const override; + + protected: + /// @param fs File system the data table underneath is read through. + /// @param table_path Root path of the data table this system table belongs to. + SystemTable(const std::shared_ptr& fs, const std::string& table_path); + + /// Loaders call this once the object exists, because `ArrowSchema()` is virtual and cannot + /// run from a constructor: it fills the schema the base hands out with this table's own + /// read-only `SystemTableSchema`. + Status AdoptOwnSchema(); + + friend class SystemTableLoader; + friend class GlobalSystemTableLoader; }; /// Loads system table implementations from parsed table identifiers or table paths. diff --git a/src/paimon/core/table/system/system_table_test.cpp b/src/paimon/core/table/system/system_table_test.cpp index 7591e53bf..49cf6838f 100644 --- a/src/paimon/core/table/system/system_table_test.cpp +++ b/src/paimon/core/table/system/system_table_test.cpp @@ -249,7 +249,7 @@ TEST(SystemTableTest, TestNewReadPropagatesWarmupLevel) { AuditLogSystemTable audit_log(/*fs=*/nullptr, "/tmp/table", table_schema, options); BinlogSystemTable binlog(/*fs=*/nullptr, "/tmp/table", table_schema, options); - ReadOptimizedSystemTable read_optimized("/tmp/table", table_schema, options); + ReadOptimizedSystemTable read_optimized(/*fs=*/nullptr, "/tmp/table", table_schema, options); for (WarmupLevel level : {WarmupLevel::NONE, WarmupLevel::RAW, WarmupLevel::DECODED}) { ReadContextBuilder builder("/tmp/table"); diff --git a/src/paimon/rest/rest_catalog.cpp b/src/paimon/rest/rest_catalog.cpp index fbd9521c4..3ff2ed756 100644 --- a/src/paimon/rest/rest_catalog.cpp +++ b/src/paimon/rest/rest_catalog.cpp @@ -390,49 +390,58 @@ Result> RestCatalog::LoadTableSchema(const Identifier& i context.catalog_options = api_->GetMergedOptions(); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr system_table, GlobalSystemTableLoader::Load(identifier.GetObjectName(), context)); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr arrow_schema, - system_table->ArrowSchema()); - return std::make_shared(std::move(arrow_schema)); + return system_table->LatestSchema(); } PAIMON_ASSIGN_OR_RAISE(bool is_system_table, identifier.IsSystemTable()); PAIMON_ASSIGN_OR_RAISE(std::optional branch, identifier.GetBranchName()); branch = NormalizeBranch(std::move(branch)); PAIMON_ASSIGN_OR_RAISE(Identifier load_identifier, ToLoadIdentifier(identifier)); if (is_system_table) { - PAIMON_ASSIGN_OR_RAISE(std::optional system_table_name, - identifier.GetSystemTableName()); - if (!system_table_name || !SystemTableLoader::IsSupported(system_table_name.value())) { - return Status::NotExist(fmt::format("{} not exist", identifier.ToString())); - } - std::string data_table_path; - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr latest_schema, - LoadDataTableSchema(load_identifier, branch, &data_table_path, - /*table_id=*/nullptr)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr system_table, + LoadSystemTable(identifier, load_identifier, branch)); if (table_path != nullptr) { - *table_path = data_table_path; - } - std::map dynamic_options; - if (branch) { - dynamic_options[Options::BRANCH] = branch.value(); + *table_path = system_table->GetRootPath(); } - // The file system is only used to build the system table, whose schema does not - // read any file, so the catalog wide one is enough here. The credentials of the - // table are applied when its rows are read, through the file system of the read or - // scan context. - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr system_table, - SystemTableLoader::Load(system_table_name.value(), fs_, data_table_path, latest_schema, - dynamic_options)); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr arrow_schema, - system_table->ArrowSchema()); - return std::make_shared(std::move(arrow_schema)); + return system_table->LatestSchema(); } PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, LoadDataTableSchema(load_identifier, branch, table_path, table_id)); return checked_pointer_cast(schema); } +Result> RestCatalog::LoadSystemTable( + const Identifier& identifier, const Identifier& load_identifier, + const std::optional& branch) const { + PAIMON_ASSIGN_OR_RAISE(std::optional system_table_name, + identifier.GetSystemTableName()); + if (!system_table_name || !SystemTableLoader::IsSupported(system_table_name.value())) { + return Status::NotExist(fmt::format("{} not exist", identifier.ToString())); + } + std::string data_table_path; + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr latest_schema, + LoadDataTableSchema(load_identifier, branch, &data_table_path, + /*table_id=*/nullptr)); + std::map dynamic_options; + if (branch) { + dynamic_options[Options::BRANCH] = branch.value(); + } + // The file system is only used to build the system table, whose schema does not + // read any file, so the catalog wide one is enough here. The credentials of the + // table are applied when its rows are read, through the file system of the read or + // scan context. + return SystemTableLoader::Load(system_table_name.value(), fs_, data_table_path, latest_schema, + dynamic_options); +} + Result> RestCatalog::GetTable(const Identifier& identifier) const { + // A `T$system` identifier opens the read-only view it names, which reads the data table it + // belongs to and answers for itself through the same `Table` interface. + PAIMON_ASSIGN_OR_RAISE(bool is_system_table, identifier.IsSystemTable()); + if (is_system_table) { + PAIMON_ASSIGN_OR_RAISE(std::optional branch, identifier.GetBranchName()); + PAIMON_ASSIGN_OR_RAISE(Identifier load_identifier, ToLoadIdentifier(identifier)); + return LoadSystemTable(identifier, load_identifier, NormalizeBranch(std::move(branch))); + } // Read UUID, path and schema together to avoid mixing table generations after recreation. std::string uuid; std::string table_path; diff --git a/src/paimon/rest/rest_catalog.h b/src/paimon/rest/rest_catalog.h index 6b7f22cda..9a4e6eb06 100644 --- a/src/paimon/rest/rest_catalog.h +++ b/src/paimon/rest/rest_catalog.h @@ -36,6 +36,7 @@ struct ArrowSchema; namespace paimon { class FileSystem; +class SystemTable; class TableSchema; /// A catalog backed by a REST catalog server. Metadata operations are delegated to @@ -127,6 +128,12 @@ class RestCatalog : public Catalog, public VersionManagedCatalog { const Identifier& data_identifier, const std::optional& branch, std::string* table_path, std::string* table_id) const; + /// Loads the system table a `T$system` identifier names, on the branch the identifier + /// carries, from the same response that carries the data table's schema and path. + Result> LoadSystemTable( + const Identifier& identifier, const Identifier& load_identifier, + const std::optional& branch) const; + /// Converts a `GetTableResponse` into a `TableSchema`, applying the table path, audit /// and branch options. static Result> ToTableSchema( diff --git a/src/paimon/rest/rest_catalog_test.cpp b/src/paimon/rest/rest_catalog_test.cpp index 48d5bf19c..c76aa4b33 100644 --- a/src/paimon/rest/rest_catalog_test.cpp +++ b/src/paimon/rest/rest_catalog_test.cpp @@ -44,6 +44,7 @@ #include "paimon/core/catalog/commit_table_request.h" #include "paimon/core/partition/partition_statistics.h" #include "paimon/core/schema/table_schema.h" +#include "paimon/core/table/system/system_table.h" #include "paimon/core/utils/snapshot_manager.h" #include "paimon/defs.h" #include "paimon/file_store_commit.h" @@ -994,9 +995,13 @@ TEST_F(RestCatalogTest, SystemTableSchema) { requests_before = state_->get_table_requests; } ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, catalog->GetTable(system_identifier)); - ASSERT_EQ("t1$options", table->Name()); + // A system table comes back as the read-only `SystemTable` it is, answering for itself + // with the kind it serves rather than the identifier it was opened by. + std::shared_ptr system_table = std::dynamic_pointer_cast(table); + ASSERT_NE(nullptr, system_table); + ASSERT_EQ("options", system_table->Name()); ASSERT_EQ((std::vector{"key", "value"}), table->LatestSchema()->FieldNames()); - ASSERT_EQ("db1.t1$options", table->Uuid()); + ASSERT_EQ("options", table->Uuid()); { std::lock_guard lock(state_->mutex); ASSERT_EQ(requests_before + 1, state_->get_table_requests); diff --git a/test/inte/read_inte_test.cpp b/test/inte/read_inte_test.cpp index 284cd362f..2c24607a2 100644 --- a/test/inte/read_inte_test.cpp +++ b/test/inte/read_inte_test.cpp @@ -4190,18 +4190,18 @@ Result> ReadGlobalSystemTable( ScanContextBuilder scan_context_builder(sys_path); scan_context_builder.SetOptions(options); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr scan_context, + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan_context, scan_context_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_scan, - system_table->NewScan(scan_context)); + system_table->NewScan(std::move(scan_context))); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr plan, table_scan->CreatePlan()); ReadContextBuilder read_context_builder(sys_path); read_context_builder.SetOptions(options); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr read_context, + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, read_context_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_read, - system_table->NewRead(read_context)); + system_table->NewRead(std::move(read_context))); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr batch_reader, table_read->CreateReader(plan->Splits())); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr result, From a2ccafd8cdef9cb656250de9e22b1b52f900a794 Mon Sep 17 00:00:00 2001 From: "yonghao.fyh" Date: Tue, 20 Oct 2026 17:49:07 +0800 Subject: [PATCH 03/17] tmp --- include/paimon/catalog/table.h | 10 +- include/paimon/file_store_commit.h | 3 +- include/paimon/file_store_write.h | 3 +- include/paimon/table/source/table_read.h | 3 +- include/paimon/table/source/table_scan.h | 3 +- src/paimon/core/table/file_store_table.h | 10 +- src/paimon/core/table/table.cpp | 594 +++++++++++++---------- src/paimon/core/table/table_test.cpp | 23 + 8 files changed, 391 insertions(+), 258 deletions(-) diff --git a/include/paimon/catalog/table.h b/include/paimon/catalog/table.h index 17b3b3d21..69abbe6d5 100644 --- a/include/paimon/catalog/table.h +++ b/include/paimon/catalog/table.h @@ -99,29 +99,29 @@ class PAIMON_EXPORT Table { } /// Creates a scan of this table from a finished scan context, the table-first entry point - /// that `TableScan::Create()` also builds through. The scan is derived from the context. + /// that `TableScan::Create()` loads the table the context names and hands the context to. /// @param context The finished scan context. /// @return The scan, or the error building it reports. virtual Result> NewScan( std::unique_ptr context) const = 0; /// Creates a read of this table from a finished read context, the table-first entry point - /// that `TableRead::Create()` also builds through. The read is derived from the context. + /// that `TableRead::Create()` loads the table the context names and hands the context to. /// @param context The finished read context. /// @return The read, or the error building it reports. virtual Result> NewRead( std::unique_ptr context) const = 0; /// Creates a write of this table from a finished write context, the table-first entry point - /// that `FileStoreWrite::Create()` also builds through. The write is derived from the context. + /// that `FileStoreWrite::Create()` loads the table the context names and hands the context to. /// @param context The finished write context. /// @return The write, or the error building it reports. virtual Result> NewWrite( std::unique_ptr context) const = 0; /// Creates a commit of this table from a finished commit context, the table-first entry point - /// that `FileStoreCommit::Create()` also builds through. The commit is derived from the - /// context. + /// that `FileStoreCommit::Create()` loads the table the context names and hands the context + /// to. /// @param context The finished commit context. /// @return The commit, or the error building it reports. virtual Result> NewCommit( diff --git a/include/paimon/file_store_commit.h b/include/paimon/file_store_commit.h index fcb094e05..581cadf2d 100644 --- a/include/paimon/file_store_commit.h +++ b/include/paimon/file_store_commit.h @@ -50,7 +50,8 @@ class CommitMessage; /// manages versions. `UseRESTCatalogCommit()` only prepares a request for the caller to send. class PAIMON_EXPORT FileStoreCommit { public: - /// Create an instance of `FileStoreCommit`. + /// Create an instance of `FileStoreCommit`: the table the context names is loaded first, then + /// the context is handed to that table's `NewCommit`. /// /// @param context A unique pointer to the `CommitContext` used for commit operations. /// diff --git a/include/paimon/file_store_write.h b/include/paimon/file_store_write.h index 8859e4f3f..5f74d6536 100644 --- a/include/paimon/file_store_write.h +++ b/include/paimon/file_store_write.h @@ -42,7 +42,8 @@ class WriteContext; /// Interface for write operations in a file store. class PAIMON_EXPORT FileStoreWrite { public: - /// Create an instance of `FileStoreWrite`. + /// Create an instance of `FileStoreWrite`: the table the context names is loaded first, then + /// the context is handed to that table's `NewWrite`. /// /// @param context A unique pointer to the `WriteContext` used for write operations. /// diff --git a/include/paimon/table/source/table_read.h b/include/paimon/table/source/table_read.h index c77b323da..2bfd84fff 100644 --- a/include/paimon/table/source/table_read.h +++ b/include/paimon/table/source/table_read.h @@ -42,7 +42,8 @@ class PAIMON_EXPORT TableRead { public: virtual ~TableRead() = default; - /// Create an instance of `TableRead`. + /// Create an instance of `TableRead`: the table the context names is loaded first, then the + /// context is handed to that table's `NewRead`. /// /// @param context A unique pointer to the `ReadContext` used for read operations. /// @return A Result containing a unique pointer to the `TableRead` instance. diff --git a/include/paimon/table/source/table_scan.h b/include/paimon/table/source/table_scan.h index 8bae74c41..39f0453c8 100644 --- a/include/paimon/table/source/table_scan.h +++ b/include/paimon/table/source/table_scan.h @@ -36,7 +36,8 @@ class ScanContext; /// A scanner interface for reading table's meta and create a plan. class PAIMON_EXPORT TableScan { public: - /// Create an instance of `TableScan`. + /// Create an instance of `TableScan`: the table the context names is loaded first, then the + /// context is handed to that table's `NewScan`. /// /// @param context A unique pointer to the `ScanContext` used for scan operations. /// @return A Result containing a unique pointer to the `TableScan` instance. diff --git a/src/paimon/core/table/file_store_table.h b/src/paimon/core/table/file_store_table.h index f02b4e528..428513e16 100644 --- a/src/paimon/core/table/file_store_table.h +++ b/src/paimon/core/table/file_store_table.h @@ -35,10 +35,12 @@ class Schema; /// its root path describes. A format table - a directory of plain data files - is a `FormatTable`, /// the other kind of `Table`. /// -/// The operators create through the context-only entry points every table serves, so a context -/// built without a table reaches the same operators: `TableScan::Create()`, `TableRead::Create()`, -/// `FileStoreWrite::Create()` and `FileStoreCommit::Create()`, which read the table's schema to -/// dispatch on its type. +/// Its `NewScan`/`NewRead`/`NewWrite`/`NewCommit` build the operators here, from the context plus +/// the schema the table was constructed with, so they serve the table as it was loaded. The +/// context-only entry points - `TableScan::Create()`, `TableRead::Create()`, +/// `FileStoreWrite::Create()` and `FileStoreCommit::Create()` - load the table the context names +/// and hand the context to that table's `NewXxx`, which is how a context built without a table +/// reaches the same operators. class FileStoreTable : public Table { public: /// The constructor a catalog builds a managed table through, so a `shared_ptr` can hold one; diff --git a/src/paimon/core/table/table.cpp b/src/paimon/core/table/table.cpp index 3b50d2e85..f9cf78545 100644 --- a/src/paimon/core/table/table.cpp +++ b/src/paimon/core/table/table.cpp @@ -209,94 +209,166 @@ Result HasHistoricalPrimaryKeyBTreeDefinition( return false; } -} // namespace - -Table::Table(const std::shared_ptr& file_system, const std::string& root_path, - const std::shared_ptr& schema, const Identifier& identifier, - const std::string& uuid) - : file_system_(file_system), - root_path_(root_path), - schema_(schema), - identifier_(identifier), - uuid_(uuid) {} - -Result> FileStoreTable::NewScan( - std::unique_ptr context) const { - return TableScan::Create(std::move(context)); -} - -Result> FileStoreTable::NewRead( - std::unique_ptr context) const { - return TableRead::Create(std::move(context)); -} - -Result> FileStoreTable::NewWrite( - std::unique_ptr context) const { - return FileStoreWrite::Create(std::move(context)); -} - -Result> FileStoreTable::NewCommit( - std::unique_ptr context) const { - return FileStoreCommit::Create(std::move(context)); +/// The tail every `TableOf` ends with: the schema one of them read says what the path names. A +/// format table is handed back as the `FormatTable` it is, everything else as the +/// `FileStoreTable` a managed table is, carrying the schema for its `NewXxx` to build from. A +/// null schema is handed over as it is: what to say about a table that is not there is the +/// managed path's to decide, and it says it in more detail. +Result> TableOfLoadedSchema( + const std::shared_ptr& file_system, const std::string& path, + const std::string& branch, const std::map& options, + const std::shared_ptr& latest_schema, const std::optional& identifier, + const std::string& uuid = "") { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr format_table, + TryLoadFormatTable(file_system, path, branch, options, latest_schema)); + if (format_table != nullptr) { + return std::shared_ptr
(std::move(format_table)); + } + // A context names a path, never a catalog identifier, so the table is known by the directory + // it sits in - the same convention `TryLoadFormatTable` follows above. + Identifier table_identifier = + identifier.value_or(Identifier(Identifier::kUnknownDatabase, PathUtil::GetName(path))); + return std::make_shared(file_system, path, latest_schema, table_identifier, + uuid); } -Result> FileStoreWrite::Create(std::unique_ptr ctx) { - if (ctx == nullptr) { - return Status::Invalid("write context is null pointer"); +/// The table a scan context names, read from what the context carries: a format table the +/// builder was handed, a system table behind a `T$system` path, or the table - format or +/// managed - the schema under the context's path says is there. +Result> TableOf(const ScanContext& ctx) { + // A table the caller already loaded says what it is, so nothing is read to find out. + if (ctx.GetFormatTable() != nullptr) { + return std::shared_ptr
(ctx.GetFormatTable()); } - if (ctx->GetMemoryPool() == nullptr) { - return Status::Invalid("memory pool is null pointer"); + PAIMON_ASSIGN_OR_RAISE(CoreOptions tmp_options, + CoreOptions::FromMap(ctx.GetOptions(), ctx.GetSpecificFileSystem(), {})); + PAIMON_ASSIGN_OR_RAISE(std::optional system_table_path, + SystemTableLoader::TryParsePath(ctx.GetPath())); + if (system_table_path) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr system_table, + SystemTableLoader::LoadFromPath(tmp_options.GetFileSystem(), + ctx.GetPath(), ctx.GetOptions())); + return std::shared_ptr
(std::move(system_table)); } - if (ctx->GetExecutor() == nullptr) { - return Status::Invalid("executor is null pointer"); + // A format table is planned by listing directories, so it never reaches the manifest path + // below. One `TableScan` interface serves both, as Java Paimon serves both through one + // `ReadBuilder`. + const std::string branch = BranchManager::NormalizeBranch(tmp_options.GetBranch()); + std::shared_ptr latest_schema; + if (branch == BranchManager::DEFAULT_MAIN_BRANCH && ctx.GetSpecificTableSchema()) { + // Handing the schema over only saves reading it. A context always names a table path and + // this library writes the metadata under it, so `schema` and `branch` stay metadata. + PAIMON_ASSIGN_OR_RAISE(latest_schema, + TableSchema::CreateFromJson(ctx.GetSpecificTableSchema().value())); + } else { + SchemaManager schema_manager(tmp_options.GetFileSystem(), ctx.GetPath(), branch); + PAIMON_ASSIGN_OR_RAISE(std::optional> latest, + schema_manager.Latest()); + if (latest) { + latest_schema = *latest; + } } + return TableOfLoadedSchema(tmp_options.GetFileSystem(), ctx.GetPath(), branch, ctx.GetOptions(), + latest_schema, ctx.GetIdentifier()); +} +/// The table a read context names, read from what the context carries: a format table the +/// builder was handed, a system table behind a `T$system` path, or the table - format or +/// managed - the schema under the context's path says is there. +Result> TableOf(const ReadContext& ctx) { // A table the caller already loaded says what it is, so nothing is read to find out. - if (ctx->GetFormatTable() != nullptr) { - return ctx->GetFormatTable()->NewWrite(std::move(ctx)); + if (ctx.GetFormatTable() != nullptr) { + return std::shared_ptr
(ctx.GetFormatTable()); + } + PAIMON_ASSIGN_OR_RAISE(CoreOptions tmp_core_options, + CoreOptions::FromMap(ctx.GetOptions(), ctx.GetSpecificFileSystem(), + ctx.GetFileSystemSchemeToIdentifierMap())); + PAIMON_ASSIGN_OR_RAISE(std::optional system_table_path, + SystemTableLoader::TryParsePath(ctx.GetPath())); + if (system_table_path) { + if (PredicateUtils::ContainsFullTextMatch(context->GetPredicate())) { + return Status::NotImplemented( + "FullTextMatch predicate execution is not implemented yet"); + } + if (ctx.HasVectorOrFullTextSearch()) { + return Status::NotImplemented( + "VectorSearch or FullTextSearch is not supported for system tables"); + } + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr system_table, + SystemTableLoader::LoadFromPath(tmp_core_options.GetFileSystem(), + ctx.GetPath(), ctx.GetOptions())); + return std::shared_ptr
(std::move(system_table)); } - PAIMON_RETURN_NOT_OK(CheckVersionManagementImplemented(ctx->GetCatalog())); - std::shared_ptr specific_fs = ctx->GetSpecificFileSystem(); - if (specific_fs == nullptr && ctx->GetFileSystemSchemeToIdentifierMap().empty() && - ctx->GetCatalog() != nullptr) { - if (!ctx->GetIdentifier()) { + // A format table has no manifest and a file another engine wrote carries no field ids, so + // which files there are and how a row is put back together is its own. How a file is opened + // is not: both paths go through `DataFileReaderFactory`. + const std::string branch = ctx.GetBranch(); + std::shared_ptr latest_schema; + if (branch == BranchManager::DEFAULT_MAIN_BRANCH && ctx.GetSpecificTableSchema()) { + // Handing the schema over only saves reading it. A context always names a table path and + // this library writes the metadata under it, so `schema` and `branch` stay metadata. + PAIMON_ASSIGN_OR_RAISE(latest_schema, + TableSchema::CreateFromJson(ctx.GetSpecificTableSchema().value())); + } else { + SchemaManager schema_manager(tmp_core_options.GetFileSystem(), ctx.GetPath(), branch); + PAIMON_ASSIGN_OR_RAISE(std::optional> latest, + schema_manager.Latest()); + if (latest) { + latest_schema = *latest; + } + } + return TableOfLoadedSchema(tmp_core_options.GetFileSystem(), ctx.GetPath(), branch, + ctx.GetOptions(), latest_schema, ctx.GetIdentifier()); +} + +/// The table a write context names, read from what the context carries: a format table the +/// builder was handed, or the table - format or managed - the schema under the context's root +/// path says is there. +Result> TableOf(const WriteContext& ctx) { + // A table the caller already loaded says what it is, so nothing is read to find out. + if (ctx.GetFormatTable() != nullptr) { + return std::shared_ptr
(ctx.GetFormatTable()); + } + PAIMON_RETURN_NOT_OK(CheckVersionManagementImplemented(ctx.GetCatalog())); + std::shared_ptr specific_fs = ctx.GetSpecificFileSystem(); + if (specific_fs == nullptr && ctx.GetFileSystemSchemeToIdentifierMap().empty() && + ctx.GetCatalog() != nullptr) { + if (!ctx.GetIdentifier()) { return Status::Invalid("a catalog write requires a table identifier"); } // A catalog issuing per-table temporary credentials only hands them out through // GetTableFileSystem, so the write uses the credentials of this table. PAIMON_ASSIGN_OR_RAISE(specific_fs, - ctx->GetCatalog()->GetTableFileSystem(ctx->GetIdentifier().value())); + ctx.GetCatalog()->GetTableFileSystem(ctx.GetIdentifier().value())); } PAIMON_ASSIGN_OR_RAISE(CoreOptions tmp_options, - CoreOptions::FromMap(ctx->GetOptions(), specific_fs, - ctx->GetFileSystemSchemeToIdentifierMap())); - std::string branch = ctx->GetBranch(); + CoreOptions::FromMap(ctx.GetOptions(), specific_fs, + ctx.GetFileSystemSchemeToIdentifierMap())); + std::string branch = ctx.GetBranch(); std::optional catalog_identifier; - if (ctx->GetIdentifier()) { - PAIMON_ASSIGN_OR_RAISE( - Identifier branch_identifier, - CatalogUtils::BranchIdentifier(ctx->GetIdentifier().value(), branch)); + if (ctx.GetIdentifier()) { + PAIMON_ASSIGN_OR_RAISE(Identifier branch_identifier, + CatalogUtils::BranchIdentifier(ctx.GetIdentifier().value(), branch)); catalog_identifier.emplace(std::move(branch_identifier)); } // Only the main branch may have a current schema the catalog alone holds. A branch publishes // its own under `branch/branch-`, which is where the load below and a read of the branch // both read it, so a catalog answering for no branch schema does not fail a write to one. std::optional catalog_table_schema; - if (ctx->GetCatalog() != nullptr) { + if (ctx.GetCatalog() != nullptr) { if (!catalog_identifier) { return Status::Invalid("a catalog write requires a table identifier"); } if (BranchManager::IsMainBranch(branch)) { PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, - ctx->GetCatalog()->LoadTableSchema(catalog_identifier.value())); + ctx.GetCatalog()->LoadTableSchema(catalog_identifier.value())); PAIMON_ASSIGN_OR_RAISE(catalog_table_schema, schema->GetJsonSchema()); } } // A format table writes plain data files into a directory, so it never reaches the manifest // path below. - auto schema_manager = - std::make_shared(tmp_options.GetFileSystem(), ctx->GetRootPath(), branch); + SchemaManager schema_manager(tmp_options.GetFileSystem(), ctx.GetRootPath(), branch); std::shared_ptr latest_schema; if (branch == BranchManager::DEFAULT_MAIN_BRANCH && catalog_table_schema) { // Handing the schema over only saves reading it. A context always names a table path and @@ -304,22 +376,136 @@ Result> FileStoreWrite::Create(std::unique_ptr> latest, - schema_manager->Latest()); + schema_manager.Latest()); if (latest) { latest_schema = *latest; } } - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr format_table, - TryLoadFormatTable(tmp_options.GetFileSystem(), ctx->GetRootPath(), - branch, ctx->GetOptions(), latest_schema)); - if (format_table != nullptr) { - return format_table->NewWrite(std::move(ctx)); + return TableOfLoadedSchema(tmp_options.GetFileSystem(), ctx.GetRootPath(), branch, + ctx.GetOptions(), latest_schema, ctx.GetIdentifier()); +} + +/// The table a commit context names, read from what the context carries: a format table the +/// builder was handed, or the table - format or managed - the schema under the context's root +/// path says is there. +Result> TableOf(const CommitContext& ctx) { + // A table the caller already loaded says what it is, so nothing is read to find out. + if (ctx.GetFormatTable() != nullptr) { + return std::shared_ptr
(ctx.GetFormatTable()); + } + PAIMON_RETURN_NOT_OK(CheckVersionManagementImplemented(ctx.GetCatalog())); + // Use catalog credentials unless the caller supplied a file system. + std::shared_ptr specific_fs = ctx.GetSpecificFileSystem(); + if (specific_fs == nullptr && ctx.GetCatalog() != nullptr) { + if (!ctx.GetIdentifier()) { + return Status::Invalid("a catalog commit requires a table identifier"); + } + // Use the credentials of this table, which a catalog issuing per-table temporary + // ones only hands out through GetTableFileSystem. + PAIMON_ASSIGN_OR_RAISE(specific_fs, + ctx.GetCatalog()->GetTableFileSystem(ctx.GetIdentifier().value())); + } + PAIMON_ASSIGN_OR_RAISE(CoreOptions tmp_options, + CoreOptions::FromMap(ctx.GetOptions(), specific_fs)); + const std::string& root_path = ctx.GetRootPath(); + // The branch the context was built for, already validated by `CommitContextBuilder`. + PAIMON_ASSIGN_OR_RAISE(std::string branch, BranchManager::ResolveBranch( + ctx.GetIdentifier(), ctx.GetOptions(), + /*explicit_branch=*/std::nullopt, "commit")); + std::optional catalog_identifier; + if (ctx.GetIdentifier()) { + PAIMON_ASSIGN_OR_RAISE(Identifier branch_identifier, + CatalogUtils::BranchIdentifier(ctx.GetIdentifier().value(), branch)); + catalog_identifier.emplace(std::move(branch_identifier)); + } + // Catalog-managed schemas may be absent from the table directory, but only on the main branch: + // a branch publishes its own under `branch/branch-`, which is where the load below and a + // read of the branch both read it, so a catalog answering for no branch schema does not fail + // a commit to one. + std::optional catalog_table_schema; + if (ctx.GetCatalog() != nullptr) { + if (!catalog_identifier) { + return Status::Invalid("a catalog commit requires a table identifier"); + } + if (BranchManager::IsMainBranch(branch)) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr loaded_schema, + ctx.GetCatalog()->LoadTableSchema(catalog_identifier.value())); + PAIMON_ASSIGN_OR_RAISE(std::string schema_json, loaded_schema->GetJsonSchema()); + catalog_table_schema = std::move(schema_json); + } + } + // A format table commits by renaming files into place, so it never reaches the snapshot path + // below. The managed path here reads the commit's branch, so this reads the same one: the two + // must not dispatch on different schemas. + SchemaManager schema_manager(tmp_options.GetFileSystem(), root_path, branch); + std::shared_ptr latest_schema; + if (branch == BranchManager::DEFAULT_MAIN_BRANCH && catalog_table_schema) { + // Handing the schema over only saves reading it. A context always names a table path and + // this library writes the metadata under it, so `schema` and `branch` stay metadata. + PAIMON_ASSIGN_OR_RAISE(latest_schema, + TableSchema::CreateFromJson(catalog_table_schema.value())); + } else { + PAIMON_ASSIGN_OR_RAISE(std::optional> latest, + schema_manager.Latest()); + if (latest) { + latest_schema = *latest; + } } - // The schema the branch read above through `schema_manager`, rather than a second read of - // the same file. + return TableOfLoadedSchema(tmp_options.GetFileSystem(), root_path, branch, ctx.GetOptions(), + latest_schema, ctx.GetIdentifier(), ctx.GetTableId().value_or("")); +} + +} // namespace + +Table::Table(const std::shared_ptr& file_system, const std::string& root_path, + const std::shared_ptr& schema, const Identifier& identifier, + const std::string& uuid) + : file_system_(file_system), + root_path_(root_path), + schema_(schema), + identifier_(identifier), + uuid_(uuid) {} + +Result> FileStoreTable::NewWrite( + std::unique_ptr ctx) const { + if (ctx == nullptr) { + return Status::Invalid("write context is null pointer"); + } + if (ctx->GetMemoryPool() == nullptr) { + return Status::Invalid("memory pool is null pointer"); + } + if (ctx->GetExecutor() == nullptr) { + return Status::Invalid("executor is null pointer"); + } + PAIMON_RETURN_NOT_OK(CheckVersionManagementImplemented(ctx->GetCatalog())); + std::shared_ptr specific_fs = ctx->GetSpecificFileSystem(); + if (specific_fs == nullptr && ctx->GetFileSystemSchemeToIdentifierMap().empty() && + ctx->GetCatalog() != nullptr) { + if (!ctx->GetIdentifier()) { + return Status::Invalid("a catalog write requires a table identifier"); + } + // A catalog issuing per-table temporary credentials only hands them out through + // GetTableFileSystem, so the write uses the credentials of this table. + PAIMON_ASSIGN_OR_RAISE(specific_fs, + ctx->GetCatalog()->GetTableFileSystem(ctx->GetIdentifier().value())); + } + PAIMON_ASSIGN_OR_RAISE(CoreOptions tmp_options, + CoreOptions::FromMap(ctx->GetOptions(), specific_fs, + ctx->GetFileSystemSchemeToIdentifierMap())); + std::string branch = ctx->GetBranch(); + std::optional catalog_identifier; + if (ctx->GetIdentifier()) { + PAIMON_ASSIGN_OR_RAISE( + Identifier branch_identifier, + CatalogUtils::BranchIdentifier(ctx->GetIdentifier().value(), branch)); + catalog_identifier.emplace(std::move(branch_identifier)); + } + auto schema_manager = + std::make_shared(tmp_options.GetFileSystem(), ctx->GetRootPath(), branch); + // The schema the table was constructed with, so a write serves the table as it was loaded + // rather than reading the schema again. + std::shared_ptr latest_schema = std::static_pointer_cast(schema_); if (latest_schema == nullptr) { return Status::Invalid(fmt::format("not found latest schema in branch {}", branch)); } @@ -586,6 +772,21 @@ Result> FileStoreWrite::Create(std::unique_ptr> FileStoreWrite::Create(std::unique_ptr ctx) { + if (ctx == nullptr) { + return Status::Invalid("write context is null pointer"); + } + if (ctx->GetMemoryPool() == nullptr) { + return Status::Invalid("memory pool is null pointer"); + } + if (ctx->GetExecutor() == nullptr) { + return Status::Invalid("executor is null pointer"); + } + // First the table the context names, then the context handed to that table's `NewWrite`. + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, TableOf(*ctx)); + return table->NewWrite(std::move(ctx)); +} + namespace { class TableScanImpl { @@ -691,71 +892,6 @@ class TableScanImpl { } }; -/// @param loaded_schema The table's schema, when the caller already read it, or null when it has -/// to be read here. `TableScan::Create()` reads one to dispatch on its table type. -Result> NewDataTableScan( - const std::shared_ptr& context, const std::shared_ptr& loaded_schema); - -} // namespace - -Result> TableScan::Create(std::unique_ptr context) { - if (context == nullptr) { - return Status::Invalid("scan context is null pointer"); - } - if (context->GetMemoryPool() == nullptr) { - return Status::Invalid("memory pool is null pointer"); - } - if (context->GetExecutor() == nullptr) { - return Status::Invalid("executor is null pointer"); - } - - // A table the caller already loaded says what it is, so nothing is read to find out. - if (context->GetFormatTable() != nullptr) { - return context->GetFormatTable()->NewScan(std::move(context)); - } - // load schema - PAIMON_ASSIGN_OR_RAISE( - CoreOptions tmp_options, - CoreOptions::FromMap(context->GetOptions(), context->GetSpecificFileSystem(), {})); - PAIMON_ASSIGN_OR_RAISE(std::optional system_table_path, - SystemTableLoader::TryParsePath(context->GetPath())); - if (system_table_path) { - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr system_table, - SystemTableLoader::LoadFromPath(tmp_options.GetFileSystem(), context->GetPath(), - context->GetOptions())); - return system_table->NewScan(std::move(context)); - } - // A format table is planned by listing directories, so it never reaches the manifest path - // below. One `TableScan` interface serves both, as Java Paimon serves both through one - // `ReadBuilder`. - const std::string branch = BranchManager::NormalizeBranch(tmp_options.GetBranch()); - std::shared_ptr latest_schema; - if (branch == BranchManager::DEFAULT_MAIN_BRANCH && context->GetSpecificTableSchema()) { - // Handing the schema over only saves reading it. A context always names a table path and - // this library writes the metadata under it, so `schema` and `branch` stay metadata. - PAIMON_ASSIGN_OR_RAISE( - latest_schema, TableSchema::CreateFromJson(context->GetSpecificTableSchema().value())); - } else { - SchemaManager schema_manager(tmp_options.GetFileSystem(), context->GetPath(), branch); - PAIMON_ASSIGN_OR_RAISE(std::optional> latest, - schema_manager.Latest()); - if (latest) { - latest_schema = *latest; - } - } - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr format_table, - TryLoadFormatTable(tmp_options.GetFileSystem(), context->GetPath(), - branch, context->GetOptions(), latest_schema)); - if (format_table != nullptr) { - return format_table->NewScan(std::move(context)); - } - // With the schema already read above, so the managed path does not read it again. - return NewDataTableScan(std::move(context), latest_schema); -} - -namespace { - Status ValidateRealtimeScan(const TableSchema& table_schema, const CoreOptions& core_options, const ScanContext& context, bool read_optimized) { if (!context.GetRealtimeContext()) { @@ -798,12 +934,22 @@ Status ValidateRealtimeScan(const TableSchema& table_schema, const CoreOptions& return Status::OK(); } -/// @param loaded_schema The schema `TableScan::Create()` already read while dispatching on the -/// table type, or null when neither the caller's json nor the branch's own directory had -/// one. -Result> NewDataTableScan( - const std::shared_ptr& context, - const std::shared_ptr& loaded_schema) { +} // namespace + +Result> FileStoreTable::NewScan( + std::unique_ptr context) const { + if (context == nullptr) { + return Status::Invalid("scan context is null pointer"); + } + if (context->GetMemoryPool() == nullptr) { + return Status::Invalid("memory pool is null pointer"); + } + if (context->GetExecutor() == nullptr) { + return Status::Invalid("executor is null pointer"); + } + // The schema the table was constructed with, so a scan serves the table as it was loaded + // rather than reading the schema again. + std::shared_ptr loaded_schema = std::static_pointer_cast(schema_); if (loaded_schema == nullptr) { return Status::Invalid("not found latest schema"); } @@ -906,12 +1052,25 @@ Result> NewDataTableScan( return batch_scan; } -} // namespace +Result> TableScan::Create(std::unique_ptr context) { + if (context == nullptr) { + return Status::Invalid("scan context is null pointer"); + } + if (context->GetMemoryPool() == nullptr) { + return Status::Invalid("memory pool is null pointer"); + } + if (context->GetExecutor() == nullptr) { + return Status::Invalid("executor is null pointer"); + } + // First the table the context names, then the context handed to that table's `NewScan`. + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, TableOf(*context)); + return table->NewScan(std::move(context)); +} namespace { /// @param loaded_schema The schema of `branch`, when the caller already read it, or null when it -/// has to be read here: `TableRead::Create()` reads one to dispatch on the table type, and +/// has to be read here: `FileStoreTable::NewRead()` hands the table's own schema over, and /// reading it again would cost a second listing and read. Result> CreateInternalReadContext( const std::shared_ptr& context, const std::string& branch, @@ -987,15 +1146,30 @@ Result> CreateTableRead( return KeyValueTableRead::Create(path_factory, internal_context, memory_pool, executor); } -Result> NewDataTableRead( - const std::shared_ptr& context, - const std::shared_ptr& loaded_schema) { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr internal_context, - CreateInternalReadContext(context, context->GetBranch(), loaded_schema)); +} // namespace +Result> FileStoreTable::NewRead( + std::unique_ptr context) const { + std::shared_ptr read_context = std::move(context); + if (read_context == nullptr) { + return Status::Invalid("read context is null pointer"); + } + if (read_context->GetMemoryPool() == nullptr) { + return Status::Invalid("memory pool is null pointer"); + } + if (read_context->GetExecutor() == nullptr) { + return Status::Invalid("executor is null pointer"); + } + // The schema the table was constructed with, so a read serves the table as it was loaded + // rather than reading the schema again. + std::shared_ptr loaded_schema = std::static_pointer_cast(schema_); PAIMON_ASSIGN_OR_RAISE( - std::unique_ptr table_read, - CreateTableRead(internal_context, context->GetMemoryPool(), context->GetExecutor())); + std::shared_ptr internal_context, + CreateInternalReadContext(read_context, read_context->GetBranch(), loaded_schema)); + + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_read, + CreateTableRead(internal_context, read_context->GetMemoryPool(), + read_context->GetExecutor())); std::optional scan_fallback_branch = internal_context->GetCoreOptions().GetScanFallbackBranch(); @@ -1006,19 +1180,17 @@ Result> NewDataTableRead( PAIMON_ASSIGN_OR_RAISE( std::shared_ptr fallback_context, - CreateInternalReadContext(context, /*branch=*/scan_fallback_branch.value(), + CreateInternalReadContext(read_context, /*branch=*/scan_fallback_branch.value(), /*loaded_schema=*/nullptr)); - PAIMON_ASSIGN_OR_RAISE( - std::unique_ptr fallback_table_read, - CreateTableRead(fallback_context, context->GetMemoryPool(), context->GetExecutor())); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr fallback_table_read, + CreateTableRead(fallback_context, read_context->GetMemoryPool(), + read_context->GetExecutor())); const std::shared_ptr& arrow_pool = internal_context->GetArrowMemoryPool(); return std::make_unique(std::move(table_read), std::move(fallback_table_read), arrow_pool); } -} // namespace - Result> TableRead::Create(std::unique_ptr ctx) { if (ctx == nullptr) { return Status::Invalid("read context is null pointer"); @@ -1029,57 +1201,9 @@ Result> TableRead::Create(std::unique_ptrGetExecutor() == nullptr) { return Status::Invalid("executor is null pointer"); } - // A table the caller already loaded says what it is, so nothing is read to find out. - if (ctx->GetFormatTable() != nullptr) { - return ctx->GetFormatTable()->NewRead(std::move(ctx)); - } - PAIMON_ASSIGN_OR_RAISE(CoreOptions tmp_core_options, - CoreOptions::FromMap(ctx->GetOptions(), ctx->GetSpecificFileSystem(), - ctx->GetFileSystemSchemeToIdentifierMap())); - PAIMON_ASSIGN_OR_RAISE(std::optional system_table_path, - SystemTableLoader::TryParsePath(ctx->GetPath())); - if (system_table_path) { - if (PredicateUtils::ContainsFullTextMatch(ctx->GetPredicate())) { - return Status::NotImplemented( - "FullTextMatch predicate execution is not implemented yet"); - } - if (ctx->HasVectorOrFullTextSearch()) { - return Status::NotImplemented( - "VectorSearch or FullTextSearch is not supported for system tables"); - } - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr system_table, - SystemTableLoader::LoadFromPath(tmp_core_options.GetFileSystem(), - ctx->GetPath(), ctx->GetOptions())); - return system_table->NewRead(std::move(ctx)); - } - - // A format table has no manifest and a file another engine wrote carries no field ids, so - // which files there are and how a row is put back together is its own. How a file is opened - // is not: both paths go through `DataFileReaderFactory`. - const std::string branch = ctx->GetBranch(); - std::shared_ptr latest_schema; - if (branch == BranchManager::DEFAULT_MAIN_BRANCH && ctx->GetSpecificTableSchema()) { - // Handing the schema over only saves reading it. A context always names a table path and - // this library writes the metadata under it, so `schema` and `branch` stay metadata. - PAIMON_ASSIGN_OR_RAISE(latest_schema, - TableSchema::CreateFromJson(ctx->GetSpecificTableSchema().value())); - } else { - SchemaManager schema_manager(tmp_core_options.GetFileSystem(), ctx->GetPath(), branch); - PAIMON_ASSIGN_OR_RAISE(std::optional> latest, - schema_manager.Latest()); - if (latest) { - latest_schema = *latest; - } - } - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr format_table, - TryLoadFormatTable(tmp_core_options.GetFileSystem(), ctx->GetPath(), - branch, ctx->GetOptions(), latest_schema)); - if (format_table != nullptr) { - return format_table->NewRead(std::move(ctx)); - } - - // With the schema already read above, so the managed path does not read it again. - return NewDataTableRead(std::move(ctx), latest_schema); + // First the table the context names, then the context handed to that table's `NewRead`. + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, TableOf(*ctx)); + return table->NewRead(std::move(ctx)); } namespace { @@ -1157,8 +1281,8 @@ Result> NewSnapshotCommit( } // namespace -Result> FileStoreCommit::Create( - std::unique_ptr ctx) { +Result> FileStoreTable::NewCommit( + std::unique_ptr ctx) const { if (ctx == nullptr) { return Status::Invalid("commit context is null pointer"); } @@ -1168,12 +1292,6 @@ Result> FileStoreCommit::Create( if (ctx->GetExecutor() == nullptr) { return Status::Invalid("executor is null pointer"); } - - // A table the caller already loaded says what it is, so nothing is read to find out. - if (ctx->GetFormatTable() != nullptr) { - return ctx->GetFormatTable()->NewCommit(std::move(ctx)); - } - PAIMON_RETURN_NOT_OK(CheckVersionManagementImplemented(ctx->GetCatalog())); // Use catalog credentials unless the caller supplied a file system. std::shared_ptr specific_fs = ctx->GetSpecificFileSystem(); @@ -1200,22 +1318,13 @@ Result> FileStoreCommit::Create( CatalogUtils::BranchIdentifier(ctx->GetIdentifier().value(), branch)); catalog_identifier.emplace(std::move(branch_identifier)); } - // Catalog-managed schemas may be absent from the table directory, but only on the main branch: - // a branch publishes its own under `branch/branch-`, which is where the load below and a - // read of the branch both read it, so a catalog answering for no branch schema does not fail - // a commit to one. - std::optional catalog_table_schema; + // The id a snapshot records the catalog's current schema by, read through the loader below + // rather than trusted from a schema this commit will not re-read. FileStoreCommitImpl::SchemaIdLoader schema_id_loader; if (ctx->GetCatalog() != nullptr) { if (!catalog_identifier) { return Status::Invalid("a catalog commit requires a table identifier"); } - if (BranchManager::IsMainBranch(branch)) { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr loaded_schema, - ctx->GetCatalog()->LoadTableSchema(catalog_identifier.value())); - PAIMON_ASSIGN_OR_RAISE(std::string schema_json, loaded_schema->GetJsonSchema()); - catalog_table_schema = std::move(schema_json); - } std::shared_ptr catalog = ctx->GetCatalog(); Identifier identifier = catalog_identifier.value(); schema_id_loader = [catalog, identifier]() -> Result { @@ -1230,32 +1339,11 @@ Result> FileStoreCommit::Create( return data_schema->Id(); }; } - // A format table commits by renaming files into place, so it never reaches the snapshot path - // below. The managed path here reads the commit's branch, so this reads the same one: the two - // must not dispatch on different schemas. auto schema_manager = std::make_shared(tmp_options.GetFileSystem(), root_path, branch); - std::shared_ptr latest_schema; - if (branch == BranchManager::DEFAULT_MAIN_BRANCH && catalog_table_schema) { - // Handing the schema over only saves reading it. A context always names a table path and - // this library writes the metadata under it, so `schema` and `branch` stay metadata. - PAIMON_ASSIGN_OR_RAISE(latest_schema, - TableSchema::CreateFromJson(catalog_table_schema.value())); - } else { - // Through the manager built above, so the read warms the cache the managed path below - // goes on to use. - PAIMON_ASSIGN_OR_RAISE(std::optional> latest, - schema_manager->Latest()); - if (latest) { - latest_schema = *latest; - } - } - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr format_table, - TryLoadFormatTable(tmp_options.GetFileSystem(), root_path, branch, - ctx->GetOptions(), latest_schema)); - if (format_table != nullptr) { - return format_table->NewCommit(std::move(ctx)); - } + // The schema the table was constructed with, so a commit serves the table as it was loaded + // rather than reading the schema again. + std::shared_ptr latest_schema = std::static_pointer_cast(schema_); if (latest_schema == nullptr) { return Status::Invalid(fmt::format("not found latest schema in branch {}", branch)); } @@ -1358,4 +1446,20 @@ Result> FileStoreCommit::Create( expire_snapshots, schema_manager, std::move(scan_supplier)); } +Result> FileStoreCommit::Create( + std::unique_ptr ctx) { + if (ctx == nullptr) { + return Status::Invalid("commit context is null pointer"); + } + if (ctx->GetMemoryPool() == nullptr) { + return Status::Invalid("memory pool is null pointer"); + } + if (ctx->GetExecutor() == nullptr) { + return Status::Invalid("executor is null pointer"); + } + // First the table the context names, then the context handed to that table's `NewCommit`. + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, TableOf(*ctx)); + return table->NewCommit(std::move(ctx)); +} + } // namespace paimon diff --git a/src/paimon/core/table/table_test.cpp b/src/paimon/core/table/table_test.cpp index 78d01e5a5..1ad8d9cd3 100644 --- a/src/paimon/core/table/table_test.cpp +++ b/src/paimon/core/table/table_test.cpp @@ -291,6 +291,29 @@ TEST(TableTest, TestNewScanCreatesATableScan) { table->NewScan(std::move(ctx))); } +TEST(TableTest, TestNewScanServesTheSchemaTheTableWasLoadedWith) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + auto fs = dir->GetFileSystem(); + const Identifier identifier("db", "tbl"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, + CreateManagedTable(fs, dir->Str(), identifier)); + + // With the schema file gone the table still serves a scan: it carries the schema it was + // loaded with, which is what a scan is built from. + SchemaManager schema_manager(fs, dir->Str()); + ASSERT_OK(fs->Delete(schema_manager.ToSchemaPath(0), /*recursive=*/false)); + ASSERT_OK_AND_ASSIGN(std::unique_ptr held_ctx, + ScanContextBuilder(dir->Str()).Finish()); + ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr scan, + table->NewScan(std::move(held_ctx))); + + // A context without a table still reads the schema from the directory, which no longer has + // one. + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, ScanContextBuilder(dir->Str()).Finish()); + ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(ctx)), "not found latest schema"); +} + TEST(TableTest, TestNewReadCreatesATableRead) { auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); From 5cac87bd9fa9020b95fa3bf6e872ed5c135d5486 Mon Sep 17 00:00:00 2001 From: "yonghao.fyh" Date: Thu, 22 Oct 2026 15:05:34 +0800 Subject: [PATCH 04/17] tmp --- include/paimon/catalog/table.h | 22 +- include/paimon/table/format/format_table.h | 35 +- src/paimon/CMakeLists.txt | 1 + .../core/catalog/file_system_catalog.cpp | 8 +- ...eted_primary_key_index_maintainer_test.cpp | 26 +- .../append_only_file_store_write_test.cpp | 22 +- .../core/operation/internal_read_context.cpp | 28 +- .../core/operation/internal_read_context.h | 14 +- src/paimon/core/schema/table_schema.cpp | 42 +- src/paimon/core/schema/table_schema.h | 13 + src/paimon/core/table/file_store.cpp | 100 +++ src/paimon/core/table/file_store.h | 125 +++ src/paimon/core/table/file_store_table.h | 59 +- src/paimon/core/table/format/format_table.cpp | 33 +- .../core/table/format/format_table_test.cpp | 26 +- .../core/table/source/table_read_test.cpp | 54 +- src/paimon/core/table/system/system_table.cpp | 5 + src/paimon/core/table/system/system_table.h | 6 + .../core/table/system/system_table_test.cpp | 14 + src/paimon/core/table/table.cpp | 717 +++++++++--------- src/paimon/core/table/table_test.cpp | 434 ++++++++++- src/paimon/rest/rest_catalog.cpp | 8 +- src/paimon/rest/rest_catalog_test.cpp | 6 +- test/inte/realtime_write_inte_test.cpp | 57 +- test/inte/scan_and_read_inte_test.cpp | 64 +- test/inte/scan_inte_test.cpp | 27 +- .../write_and_read_inte_with_index_test.cpp | 3 +- 27 files changed, 1449 insertions(+), 500 deletions(-) create mode 100644 src/paimon/core/table/file_store.cpp create mode 100644 src/paimon/core/table/file_store.h diff --git a/include/paimon/catalog/table.h b/include/paimon/catalog/table.h index 69abbe6d5..a31d3100d 100644 --- a/include/paimon/catalog/table.h +++ b/include/paimon/catalog/table.h @@ -55,13 +55,15 @@ class PAIMON_EXPORT Table { /// Returns the file system the table was opened with, which serves the table's /// files. A catalog that hands out per-table file systems opens a table with the - /// one for that table; it is null when the opener had none. + /// one for that table; it is null when the opener had none. The `NewXxx` + /// operations read the table through it. std::shared_ptr GetFileSystem() const { return file_system_; } /// Returns the root path of the table in its file system. It is empty when the /// opener had no location for the table, which a mock in a test is one case of. + /// The `NewXxx` operations serve the table from it. const std::string& GetRootPath() const { return root_path_; } @@ -74,13 +76,6 @@ class PAIMON_EXPORT Table { /// table answers database.tableName, a system table the same name as `Name()`. virtual std::string FullName() const = 0; - /// Returns the branch the table was opened for, which its identifier carries; `main` - /// when it names none or a spelling of it, which its managers are bound to. A - /// malformed object name that hides the branch is reported. - Result CurrentBranch() const { - return identifier_.GetBranchNameOrDefault(); - } - /// Returns the metastore UUID, falling back to `FullName()`. /// @note Use `CatalogUuid()` for commit requests to avoid sending a table name as an ID. std::string Uuid() const { @@ -98,6 +93,17 @@ class PAIMON_EXPORT Table { return schema_; } + /// Copies this table with `dynamic_options` on top of the options it carries, which is + /// the precedence every context builder promises for a table it was handed rather than loaded + /// itself: an option given at the call wins over the one the schema stored. + /// + /// `type` is not overridable: it is structural and is read from the schema alone. + /// + /// @param dynamic_options Options given at the call. + /// @return A result containing the copied table, or the error copying it reports. + virtual Result> Copy( + const std::map& dynamic_options) const = 0; + /// Creates a scan of this table from a finished scan context, the table-first entry point /// that `TableScan::Create()` loads the table the context names and hands the context to. /// @param context The finished scan context. diff --git a/include/paimon/table/format/format_table.h b/include/paimon/table/format/format_table.h index d15a8f6ce..88444d360 100644 --- a/include/paimon/table/format/format_table.h +++ b/include/paimon/table/format/format_table.h @@ -55,8 +55,7 @@ class FileSystem; /// loaded through the `FormatTable` constructor every context builder has, which is how a table /// whose schema lives in a metastore is reached. /// -/// Every instance lives in a `shared_ptr`, which is what its operators hand to -/// `FormatTable::Copy()`. +/// Every instance lives in a `shared_ptr`: a copy that adds nothing hands the same one back. /// /// See `docs/source/user_guide/format_table.rst` for what is not supported yet. class PAIMON_EXPORT FormatTable : public Table, public std::enable_shared_from_this { @@ -108,20 +107,7 @@ class PAIMON_EXPORT FormatTable : public Table, public std::enable_shared_from_t bool location_carries_paimon_metadata, const std::map& dynamic_options); - /// Copies `table` with `dynamic_options` on top of the options it already carries, which is - /// the precedence every context builder promises for a table it was handed rather than loaded - /// itself. `table` comes back as it is when there is nothing to add. - /// - /// `type` is not overridable: it is structural and is read from the schema alone. - /// - /// @param table The table to copy. - /// @param dynamic_options Options given at the call. - /// @return A result containing the copied table, or an error status. - static Result> Copy( - const std::shared_ptr& table, - const std::map& dynamic_options); - - ~FormatTable(); + ~FormatTable() override; std::string Name() const override { return identifier_.GetObjectName(); @@ -131,6 +117,11 @@ class PAIMON_EXPORT FormatTable : public Table, public std::enable_shared_from_t return identifier_.GetFullName(); } + /// Copies the table with `dynamic_options` on top of the options it carries, the precedence + /// every context builder promises. The same table comes back when there is nothing to add. + Result> Copy( + const std::map& dynamic_options) const override; + /// Directory the data files live in. const std::string& Location() const { return GetRootPath(); @@ -217,15 +208,21 @@ class PAIMON_EXPORT FormatTable : public Table, public std::enable_shared_from_t const std::string& file_compression, const std::string& partition_default_name, bool partition_only_value_in_path, bool location_carries_paimon_metadata); - /// This table as a `shared_ptr`, which the operators above hand to `FormatTable::Copy()`. + /// This table as a `shared_ptr`, which a copy that adds nothing hands back. /// /// `shared_from_this()` on a const method hands back a `shared_ptr`, while - /// `Copy()` takes a mutable one; the cast is safe because `Copy()` never writes to the table - /// it is given. + /// the copy below needs a mutable one; the cast is safe because the table is never written to. std::shared_ptr SharedSelf() const { return std::const_pointer_cast(shared_from_this()); } + /// Copies this table with `dynamic_options` on top of the options it carries, as a format + /// table rather than the `Table` `Copy()` hands back, which is what its operators need. The + /// same table comes back when there is nothing to add; `type` is not overridable, and + /// `Create()` keeps the schema's own. + Result> CopySelf( + const std::map& dynamic_options) const; + std::map options_; Format format_; std::string file_compression_; diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt index ad3d65f90..261987aea 100644 --- a/src/paimon/CMakeLists.txt +++ b/src/paimon/CMakeLists.txt @@ -445,6 +445,7 @@ set(PAIMON_CORE_SRCS core/stats/simple_stats_evolution.cpp core/table/table.cpp core/table/bucket_mode.cpp + core/table/file_store.cpp core/table/format/format_file_listing.cpp core/table/format/format_file_naming.cpp core/table/format/format_path_validation.cpp diff --git a/src/paimon/core/catalog/file_system_catalog.cpp b/src/paimon/core/catalog/file_system_catalog.cpp index 46131f729..36721a148 100644 --- a/src/paimon/core/catalog/file_system_catalog.cpp +++ b/src/paimon/core/catalog/file_system_catalog.cpp @@ -347,8 +347,12 @@ Result> FileSystemCatalog::GetTable(const Identifier& ide // table rather than at the first read. PAIMON_RETURN_NOT_OK(CatalogUtils::CheckManagedTableType(identifier, *latest_schema, "FileSystemCatalog::GetTable")); - return std::make_shared( - fs_, table_path, checked_pointer_cast(*latest_schema), identifier); + // The branch the schema above was read on, which the table carries so its managers read + // the same branch: `GetBranchNameOrDefault` folds any spelling of `main`. + PAIMON_ASSIGN_OR_RAISE(std::string branch, identifier.GetBranchNameOrDefault()); + return std::make_shared(fs_, table_path, + checked_pointer_cast(*latest_schema), + identifier, /*uuid=*/"", branch); } Result> FileSystemCatalog::LoadSystemTable( diff --git a/src/paimon/core/index/pk/bucketed_primary_key_index_maintainer_test.cpp b/src/paimon/core/index/pk/bucketed_primary_key_index_maintainer_test.cpp index 81803ed90..769bbf326 100644 --- a/src/paimon/core/index/pk/bucketed_primary_key_index_maintainer_test.cpp +++ b/src/paimon/core/index/pk/bucketed_primary_key_index_maintainer_test.cpp @@ -33,6 +33,7 @@ #include "gtest/gtest.h" #include "paimon/catalog/catalog.h" #include "paimon/catalog/identifier.h" +#include "paimon/catalog/table.h" #include "paimon/commit_context.h" #include "paimon/common/utils/path_util.h" #include "paimon/core/global_index/indexed_split_impl.h" @@ -473,15 +474,21 @@ TEST_P(BucketedPrimaryKeyIndexMaintainerTest, BuildsRestoresAndReplacesCompacted ASSERT_EQ(0, data_split_count); ReadContextBuilder read_builder(table_path_); - // The source build above keeps lazy decoding enabled. Disable it only for the final data-row - // assertion, whose predicate reader is outside the maintenance path covered by this test. read_builder.SetReadFieldNames({"id", "value"}) .SetPredicate(value_predicate) - .AddOption("orc.read.enable-lazy-decoding", "false") .EnablePredicateFilter(true); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); + // The source build above keeps lazy decoding enabled. Disable it only for the final data-row + // assertion, whose predicate reader is outside the maintenance path covered by this test. + // The table is copied with it off because a read through the loaded table refuses an option + // that contradicts the one it carries. + ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, Catalog::Create(directory_->Str(), {})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
loaded_table, + catalog->GetTable(Identifier("db", "table"))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
copied_table, + loaded_table->Copy({{"orc.read.enable-lazy-decoding", "false"}})); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + copied_table->NewRead(std::move(read_context))); ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_reader, table_read->CreateReader(plan->Splits())); ASSERT_OK_AND_ASSIGN(std::shared_ptr result, @@ -707,11 +714,18 @@ TEST_P(BucketedPrimaryKeyIndexMaintainerTest, ContinuesCompactionWhenIndexBuildF ReadContextBuilder read_builder(table_path_); read_builder.SetReadFieldNames({"id", "value"}) .SetPredicate(value_predicate) - .AddOption("orc.read.enable-lazy-decoding", "false") .EnablePredicateFilter(true); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); + // Like the test above, the assertion reads with lazy decoding disabled, so it reads through + // a copy of the table carrying it off: a read through the loaded table refuses an option that + // contradicts the one it carries. + ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, Catalog::Create(directory_->Str(), {})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
loaded_table, + catalog->GetTable(Identifier("db", "table"))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
copied_table, + loaded_table->Copy({{"orc.read.enable-lazy-decoding", "false"}})); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); + copied_table->NewRead(std::move(read_context))); ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_reader, table_read->CreateReader(plan->Splits())); ASSERT_OK_AND_ASSIGN(std::shared_ptr result, diff --git a/src/paimon/core/operation/append_only_file_store_write_test.cpp b/src/paimon/core/operation/append_only_file_store_write_test.cpp index bfeaefced..3ce48ee2c 100644 --- a/src/paimon/core/operation/append_only_file_store_write_test.cpp +++ b/src/paimon/core/operation/append_only_file_store_write_test.cpp @@ -36,6 +36,7 @@ #include "gtest/gtest.h" #include "paimon/catalog/catalog.h" #include "paimon/catalog/identifier.h" +#include "paimon/catalog/table.h" #include "paimon/commit_context.h" #include "paimon/common/data/binary_row.h" #include "paimon/common/data/binary_row_writer.h" @@ -513,10 +514,25 @@ TEST_F(AppendOnlyFileStoreWriteTest, TestSharedShreddingNewWriterIgnoresExisting /*commit_identifier=*/0); Commit(table_path, avro_options, avro_commit_msgs); - auto second_commit_msgs = WriteAndPrepare(table_path, logical_schema, shredding_options, R"([ + // The second batch is written through a copy of the table carrying the shredding options: + // an option on the context would contradict the table the catalog loaded, which a write + // through that table refuses. + ASSERT_OK_AND_ASSIGN(auto catalog, Catalog::Create(dir->Str(), avro_options)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
avro_table, + catalog->GetTable(Identifier("foo", "bar"))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
shredded_table, + avro_table->Copy(shredding_options)); + WriteContextBuilder shredded_builder(table_path, commit_user_); + ASSERT_OK_AND_ASSIGN(std::unique_ptr shredded_context, shredded_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto shredded_write, + shredded_table->NewWrite(std::move(shredded_context))); + ASSERT_OK(shredded_write->Write(MakeBatch(logical_schema, R"([ [2, [["c", 3], ["d", 4], ["e", 5]]] - ])", - /*commit_identifier=*/1); + ])"))); + ASSERT_OK_AND_ASSIGN(auto second_commit_msgs, + shredded_write->PrepareCommit(/*wait_compaction=*/false, + /*commit_identifier=*/1)); + ASSERT_OK(shredded_write->Close()); auto second_file_schema = ReadDataFileSchema(table_path, OnlyNewFile(second_commit_msgs), shredding_options); auto second_meta = ShreddingMeta(second_file_schema, /*field_index=*/1); diff --git a/src/paimon/core/operation/internal_read_context.cpp b/src/paimon/core/operation/internal_read_context.cpp index f36a15d71..6c0c43341 100644 --- a/src/paimon/core/operation/internal_read_context.cpp +++ b/src/paimon/core/operation/internal_read_context.cpp @@ -275,10 +275,15 @@ std::optional InternalReadContext::TryResolveSpecialFieldByName( Result> InternalReadContext::Create( const std::shared_ptr& context, const std::shared_ptr& table_schema, - const std::map& options) { - PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, - CoreOptions::FromMap(options, context->GetSpecificFileSystem(), - context->GetFileSystemSchemeToIdentifierMap())); + const std::map& options, + const std::shared_ptr& file_system, const std::string* path) { + // The file system a table reads through, or null for the read context's own, which callers + // without a table hand over. + PAIMON_ASSIGN_OR_RAISE( + CoreOptions core_options, + CoreOptions::FromMap( + options, file_system != nullptr ? file_system : context->GetSpecificFileSystem(), + context->GetFileSystemSchemeToIdentifierMap())); core_options.WithCache(context->GetCache()); if (context->HasVectorOrFullTextSearch()) { if (!core_options.FileIndexReadEnabled()) { @@ -390,10 +395,11 @@ Result> InternalReadContext::Create( return Status::Invalid("memory pool is null"); } std::shared_ptr arrow_pool = GetArrowPool(context->GetMemoryPool()); - auto internal_context = std::unique_ptr( - new InternalReadContext(context, table_schema, read_schema, core_options, arrow_pool)); + auto internal_context = std::unique_ptr(new InternalReadContext( + context, table_schema, read_schema, core_options, arrow_pool, + path != nullptr ? std::optional(*path) : std::nullopt)); PAIMON_RETURN_NOT_OK(internal_context->BindIndexScores()); - return internal_context; + return internal_context } Status InternalReadContext::BindIndexScores() { @@ -471,12 +477,14 @@ InternalReadContext::InternalReadContext(const std::shared_ptr& rea const std::shared_ptr& table_schema, const std::shared_ptr& read_schema, const CoreOptions& options, - const std::shared_ptr& arrow_pool) + const std::shared_ptr& arrow_pool, + std::optional path) : read_context_(read_context), table_schema_(table_schema), read_schema_(read_schema), arrow_pool_(arrow_pool), - options_(options) {} + options_(options), + path_(std::move(path)) {} Result> InternalReadContext::CreateWithSchema( const std::shared_ptr& original, @@ -490,7 +498,7 @@ Result> InternalReadContext::CreateWithSche // The new read_schema is the minimal column set for COUNT(*). return std::shared_ptr( new InternalReadContext(original->read_context_, original->table_schema_, new_read_schema, - original->options_, original->arrow_pool_)); + original->options_, original->arrow_pool_, original->path_)); } } // namespace paimon diff --git a/src/paimon/core/operation/internal_read_context.h b/src/paimon/core/operation/internal_read_context.h index 9de415d5f..b1619c5bc 100644 --- a/src/paimon/core/operation/internal_read_context.h +++ b/src/paimon/core/operation/internal_read_context.h @@ -21,6 +21,7 @@ #include #include #include +#include #include #include @@ -37,6 +38,7 @@ class Schema; namespace paimon { struct CatalogContext; class Executor; +class FileSystem; class MemoryPool; class Predicate; class RealtimeContext; @@ -59,7 +61,9 @@ class InternalReadContext { static Result> Create( const std::shared_ptr& read_context, const std::shared_ptr& table_schema, - const std::map& options); + const std::map& options, + const std::shared_ptr& file_system = nullptr, + const std::string* path = nullptr); /// Maps score output positions in the read schema to their logical search sources. const std::map& GetIndexScoreBindings() const { @@ -79,7 +83,7 @@ class InternalReadContext { return table_schema_; } const std::string& GetPath() const { - return read_context_->GetPath(); + return path_ ? *path_ : read_context_->GetPath(); } std::shared_ptr CreateDependencyCatalogContext() const; @@ -164,7 +168,8 @@ class InternalReadContext { const std::shared_ptr& table_schema, const std::shared_ptr& read_schema, const CoreOptions& options, - const std::shared_ptr& arrow_pool); + const std::shared_ptr& arrow_pool, + std::optional path = std::nullopt); static std::optional TryResolveSpecialFieldById(int32_t field_id, const CoreOptions& core_options, @@ -184,6 +189,9 @@ class InternalReadContext { std::shared_ptr arrow_pool_; CoreOptions options_; std::map index_score_bindings_; + /// The root path a table reads through, or null for the read context's own, which callers + /// without a table hand over. + std::optional path_; }; } // namespace paimon diff --git a/src/paimon/core/schema/table_schema.cpp b/src/paimon/core/schema/table_schema.cpp index 1d69c033a..e6151d4c7 100644 --- a/src/paimon/core/schema/table_schema.cpp +++ b/src/paimon/core/schema/table_schema.cpp @@ -194,6 +194,35 @@ TableSchema::TableSchema(int32_t version, int64_t id, const std::vector> TableSchema::MergeOptions( + const std::map& dynamic_options) const { + std::map options = options_; + for (const auto& [key, value] : dynamic_options) { + options[key] = value; + } + auto merged = std::unique_ptr( + new TableSchema(version_, id_, fields_, highest_field_id_, partition_keys_, primary_keys_, + options, comment_, time_millis_)); + // The derived members follow the merged options: a copy the options restyle must serve the + // bucket keys and the bucket count it carries, not the ones the base schema carried. + PAIMON_RETURN_NOT_OK(merged->DeriveBucketInfo()); + return merged; +} + +Status TableSchema::DeriveBucketInfo() { + PAIMON_ASSIGN_OR_RAISE(std::vector keys, TrimmedPrimaryKeys()); + + // Try to validate bucket keys + PAIMON_ASSIGN_OR_RAISE(std::vector bucket_keys, OriginalBucketKeys()); + if (bucket_keys.empty()) { + bucket_keys = keys; + } + bucket_keys_ = bucket_keys; + PAIMON_ASSIGN_OR_RAISE(num_bucket_, + OptionsUtils::GetValueFromMap(Options(), Options::BUCKET, -1)); + return Status::OK(); +} + bool TableSchema::operator==(const TableSchema& other) const { return version_ == other.version_ && fields_ == other.fields_ && partition_keys_ == other.partition_keys_ && primary_keys_ == other.primary_keys_ && @@ -332,18 +361,7 @@ Result> TableSchema::InitSchema( auto table_schema = std::unique_ptr( new TableSchema(TableSchema::CURRENT_VERSION, schema_id, fields, highest_field_id, partition_keys, primary_keys, options, comment, time_millis)); - PAIMON_ASSIGN_OR_RAISE(std::vector keys, table_schema->TrimmedPrimaryKeys()); - - // Try to validate bucket keys - PAIMON_ASSIGN_OR_RAISE(std::vector bucket_keys, - table_schema->OriginalBucketKeys()); - if (bucket_keys.empty()) { - bucket_keys = keys; - } - table_schema->bucket_keys_ = bucket_keys; - PAIMON_ASSIGN_OR_RAISE( - table_schema->num_bucket_, - OptionsUtils::GetValueFromMap(table_schema->Options(), Options::BUCKET, -1)); + PAIMON_RETURN_NOT_OK(table_schema->DeriveBucketInfo()); return table_schema; } diff --git a/src/paimon/core/schema/table_schema.h b/src/paimon/core/schema/table_schema.h index eab109cf1..aff4ee920 100644 --- a/src/paimon/core/schema/table_schema.h +++ b/src/paimon/core/schema/table_schema.h @@ -101,6 +101,15 @@ class TableSchema : public DataSchema, public Jsonizable { const std::map& Options() const override { return options_; } + + /// A copy of this schema with `dynamic_options` on top of the options it carries, which is + /// the precedence every copy of a table promises. Everything else - id, fields, keys, + /// comment - is copied as it is. + /// + /// @param dynamic_options Options given at the call, which win over the schema's own. + /// @return A result containing the merged schema, or an error status. + Result> MergeOptions( + const std::map& dynamic_options) const; const std::vector& Fields() const { return fields_; } @@ -142,6 +151,10 @@ class TableSchema : public DataSchema, public Jsonizable { Result> OriginalBucketKeys() const; + /// Derives `bucket_keys_` and `num_bucket_` from the options the schema carries: every + /// constructor path must call this, as both members serve scans and copies. + Status DeriveBucketInfo(); + static Result> AssignFieldIdsRecursively( const std::shared_ptr& field, bool set_field_id, int32_t* field_id); diff --git a/src/paimon/core/table/file_store.cpp b/src/paimon/core/table/file_store.cpp new file mode 100644 index 000000000..e70e85c1d --- /dev/null +++ b/src/paimon/core/table/file_store.cpp @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/table/file_store.h" + +#include +#include +#include +#include + +#include "paimon/catalog/catalog.h" +#include "paimon/common/utils/arrow/status_utils.h" +#include "paimon/core/catalog/version_managed_catalog.h" +#include "paimon/core/core_options.h" +#include "paimon/core/schema/schema_manager.h" +#include "paimon/core/schema/table_schema.h" +#include "paimon/core/utils/branch_manager.h" +#include "paimon/core/utils/file_store_path_factory.h" +#include "paimon/core/utils/snapshot_manager.h" +#include "paimon/result.h" +#include "paimon/status.h" + +namespace paimon { + +FileStore::FileStore(const std::shared_ptr& file_system, const std::string& root_path, + const std::shared_ptr& schema, const std::string& branch, + const std::shared_ptr& schema_manager, + const std::shared_ptr& snapshot_manager) + : file_system_(file_system), + root_path_(root_path), + schema_(schema), + branch_(BranchManager::NormalizeBranch(branch)), + schema_manager_(schema_manager), + snapshot_manager_(snapshot_manager) { + // A manager handed over points at `root_path` on `branch` already; a missing one is built + // here, so the operators always read through the managers the store carries. + if (schema_manager_ == nullptr) { + schema_manager_ = std::make_shared(file_system_, root_path_, branch_); + } + if (snapshot_manager_ == nullptr) { + snapshot_manager_ = std::make_shared(file_system_, root_path_, branch_); + } +} + +Result> FileStore::NewPathFactory( + const std::shared_ptr& arrow_schema, const CoreOptions& options, + const std::string& file_format_identifier, + const std::shared_ptr& memory_pool) const { + return MakePathFactory(root_path_, arrow_schema, schema_->PartitionKeys(), options, + file_format_identifier, memory_pool); +} + +Result> FileStore::NewSnapshotManager( + const CoreOptions& options, const std::shared_ptr& catalog, + const std::optional& catalog_identifier) const { + auto snapshot_manager = + std::make_shared(options.GetFileSystem(), root_path_, branch_); + if (VersionManagedCatalog* versioned = AsVersionManaged(catalog)) { + if (!catalog_identifier) { + return Status::Invalid("a version-managed catalog requires a table identifier"); + } + Identifier identifier = catalog_identifier.value(); + snapshot_manager->SetSnapshotLoader( + [catalog, versioned, identifier]() -> Result> { + return versioned->LoadSnapshot(identifier); + }); + } + return snapshot_manager; +} + +Result> FileStore::MakePathFactory( + const std::string& root_path, const std::shared_ptr& arrow_schema, + const std::vector& partition_keys, const CoreOptions& options, + const std::string& file_format_identifier, const std::shared_ptr& memory_pool) { + PAIMON_ASSIGN_OR_RAISE(std::vector external_paths, options.CreateExternalPaths()); + PAIMON_ASSIGN_OR_RAISE(std::optional global_index_external_path, + options.CreateGlobalIndexExternalPath()); + return FileStorePathFactory::Create( + root_path, arrow_schema, partition_keys, options.GetPartitionDefaultName(), + file_format_identifier, options.DataFilePrefix(), options.LegacyPartitionNameEnabled(), + external_paths, global_index_external_path, options.IndexFileInDataFileDir(), memory_pool); +} + +} // namespace paimon diff --git a/src/paimon/core/table/file_store.h b/src/paimon/core/table/file_store.h new file mode 100644 index 000000000..ec3bc155c --- /dev/null +++ b/src/paimon/core/table/file_store.h @@ -0,0 +1,125 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include +#include + +#include "paimon/catalog/identifier.h" +#include "paimon/core/schema/schema_manager.h" +#include "paimon/core/utils/branch_manager.h" +#include "paimon/core/utils/file_store_path_factory.h" +#include "paimon/core/utils/snapshot_manager.h" +#include "paimon/result.h" +#include "paimon/type_fwd.h" + +namespace arrow { +class Schema; +} // namespace arrow + +namespace paimon { + +class Catalog; +class CoreOptions; +class TableSchema; + +/// The storage a managed table is served through: the file system, root path, schema and branch +/// it was loaded with, and the schema and snapshot managers that read its root path on that +/// branch - the state Java Paimon's `AbstractFileStore` carries. `FileStoreTable` hands the +/// state it was loaded with to one of these and asks it back for the pieces every operator +/// opens with. +/// +/// The factory methods build a fresh instance on every call, never a cached one: a path factory +/// names each file it builds, and a snapshot manager a write or a commit arms with a catalog +/// loader is not one a scan should be served through afterwards. +class FileStore { + public: + /// @param file_system File system the table's data is accessed through. + /// @param root_path Root path of the table in the file system. + /// @param schema The schema the table was loaded with, whose partition keys name the + /// partitions the paths below are built for. + /// @param branch The branch the store serves, normalized here. + /// @param schema_manager A schema manager to carry instead of building one - the one a + /// context dispatch read the schema through, whose cache the operators then serve. + /// @param snapshot_manager A snapshot manager to carry instead of building one. + /// A null manager is built over `root_path` on `branch`, so the operators always read + /// through the managers the store carries. + FileStore(const std::shared_ptr& file_system, const std::string& root_path, + const std::shared_ptr& schema, const std::string& branch, + const std::shared_ptr& schema_manager = nullptr, + const std::shared_ptr& snapshot_manager = nullptr); + + /// The branch this store serves, as resolved when it was built. + const std::string& branch() const { + return branch_; + } + + /// The schema manager the operators read through. It carries the schema cache of `root_path` + /// on `branch()`. + const std::shared_ptr& schema_manager() const { + return schema_manager_; + } + + /// The snapshot manager a scan reads through. A write or a commit builds its own: see + /// `NewSnapshotManager`. + const std::shared_ptr& snapshot_manager() const { + return snapshot_manager_; + } + + /// The path factory an operator builds its metadata paths through: over this store's root + /// path and the partition keys of its schema. The file format identifier is the caller's to + /// name: a write names the format its level-0 files are written in, which the options may + /// set per level, while a scan, a read and a commit name the table's format. + Result> NewPathFactory( + const std::shared_ptr& arrow_schema, const CoreOptions& options, + const std::string& file_format_identifier, + const std::shared_ptr& memory_pool) const; + + /// The snapshot manager a write or a commit reads snapshots through, over this store's root + /// path and branch. The file system is the caller's to name, from the options its operator + /// serves: a table opened without one has its options build the resolving file system. A + /// catalog that version-manages the table hands its snapshots out itself, so its loader is + /// attached, the catalog captured to keep the versioned interface - and the loader with it + /// - alive. Such a catalog names the table by an identifier, which the context must have + /// supplied. + Result> NewSnapshotManager( + const CoreOptions& options, const std::shared_ptr& catalog, + const std::optional& catalog_identifier) const; + + /// The assembly the factory methods above share: a path factory over a root path and + /// partition keys named by the caller rather than the store's own, which the read path + /// needs - the schema it serves may be a fallback branch's, not the table's. + static Result> MakePathFactory( + const std::string& root_path, const std::shared_ptr& arrow_schema, + const std::vector& partition_keys, const CoreOptions& options, + const std::string& file_format_identifier, const std::shared_ptr& memory_pool); + + private: + std::shared_ptr file_system_; + std::string root_path_; + std::shared_ptr schema_; + std::string branch_; + std::shared_ptr schema_manager_; + std::shared_ptr snapshot_manager_; +}; + +} // namespace paimon diff --git a/src/paimon/core/table/file_store_table.h b/src/paimon/core/table/file_store_table.h index 428513e16..51c7c0b95 100644 --- a/src/paimon/core/table/file_store_table.h +++ b/src/paimon/core/table/file_store_table.h @@ -19,11 +19,16 @@ #pragma once +#include #include #include #include "paimon/catalog/identifier.h" #include "paimon/catalog/table.h" +#include "paimon/core/schema/schema_manager.h" +#include "paimon/core/table/file_store.h" +#include "paimon/core/utils/branch_manager.h" +#include "paimon/core/utils/snapshot_manager.h" #include "paimon/result.h" #include "paimon/type_fwd.h" @@ -36,7 +41,8 @@ class Schema; /// the other kind of `Table`. /// /// Its `NewScan`/`NewRead`/`NewWrite`/`NewCommit` build the operators here, from the context plus -/// the schema the table was constructed with, so they serve the table as it was loaded. The +/// the schema, file system, root path, branch and managers the table was constructed with, so +/// they serve the table as it was loaded. The /// context-only entry points - `TableScan::Create()`, `TableRead::Create()`, /// `FileStoreWrite::Create()` and `FileStoreCommit::Create()` - load the table the context names /// and hand the context to that table's `NewXxx`, which is how a context built without a table @@ -48,10 +54,39 @@ class FileStoreTable : public Table { /// @param file_system File system the table's data is accessed through. /// @param root_path Root path of the table in the file system. /// @param uuid Metastore UUID, or empty if unavailable. + /// @param branch The branch the table serves, which the managers below read and write on. + /// @param schema_manager A schema manager to carry instead of building one - the one a + /// context dispatch read the schema through, whose cache the operators then serve. + /// @param snapshot_manager A snapshot manager to carry instead of building one, the way + /// `Copy` hands its table's over so both share one. + /// A null manager is built over `root_path` on `branch`, so the operators always read + /// through the managers the table carries. FileStoreTable(const std::shared_ptr& file_system, const std::string& root_path, const std::shared_ptr& schema, const Identifier& identifier, - const std::string& uuid = "") - : Table(file_system, root_path, schema, identifier, uuid) {} + const std::string& uuid = "", + const std::string& branch = BranchManager::DEFAULT_MAIN_BRANCH, + const std::shared_ptr& schema_manager = nullptr, + const std::shared_ptr& snapshot_manager = nullptr); + + /// The branch this table serves, as resolved when the table was built: the schema the table + /// was loaded with came from it, and its managers point at it. + const std::string& CurrentBranch() const { + return store_->branch(); + } + + /// The schema manager the operators read through. It carries the schema cache of `root_path` + /// on `CurrentBranch()`, which every operator serves rather than each reading the schema + /// files again. + const std::shared_ptr& GetSchemaManager() const { + return store_->schema_manager(); + } + + /// The snapshot manager a scan reads through. A write or a commit builds its own: it arms + /// the manager with a catalog snapshot loader, which a shared one would keep serving to + /// scans afterwards. + const std::shared_ptr& GetSnapshotManager() const { + return store_->snapshot_manager(); + } std::string Name() const override { return identifier_.GetObjectName(); @@ -61,6 +96,19 @@ class FileStoreTable : public Table { return identifier_.GetFullName(); } + /// Copies the table with `dynamic_options` on top of the ones its schema stored, the + /// precedence `Copy` promises. The file system, root path and identifier stay as they are: + /// they name this table, which an option does not rename. + Result> Copy( + const std::map& dynamic_options) const override; + + /// Returns the table of `branch`, as Java Paimon's `switchToBranch` does. The branch switched + /// to is served as it is: its schema is read from its own directory, so the dynamic options + /// of this table are not carried over. Switching to the branch already served hands back a + /// copy of this table, schema and managers shared, and a branch with no schema does not + /// exist. + Result> SwitchToBranch(const std::string& branch) const; + Result> NewScan(std::unique_ptr context) const override; Result> NewRead(std::unique_ptr context) const override; @@ -70,6 +118,11 @@ class FileStoreTable : public Table { Result> NewCommit( std::unique_ptr context) const override; + + private: + /// The storage this table serves: the file system, root path, schema, branch and managers it + /// was loaded with, and the factories that build the pieces every operator opens with. + std::shared_ptr store_; }; } // namespace paimon diff --git a/src/paimon/core/table/format/format_table.cpp b/src/paimon/core/table/format/format_table.cpp index 53d53b967..6f2ac16a4 100644 --- a/src/paimon/core/table/format/format_table.cpp +++ b/src/paimon/core/table/format/format_table.cpp @@ -192,24 +192,26 @@ Result> FormatTable::Create( core_options.FormatTablePartitionOnlyValueInPath(), location_carries_paimon_metadata)); } -Result> FormatTable::Copy( - const std::shared_ptr& table, - const std::map& dynamic_options) { - if (table == nullptr) { - return Status::Invalid("format table requires a table to copy"); - } +Result> FormatTable::CopySelf( + const std::map& dynamic_options) const { if (dynamic_options.empty()) { - return table; + return SharedSelf(); } // The options this table already carries go back in as dynamic ones, so an option an earlier // call set is not lost to the schema's own value underneath it. - std::map options = table->options_; + std::map options = options_; for (const auto& [key, value] : dynamic_options) { options[key] = value; } - return Create(table->file_system_, table->root_path_, table->identifier_, - std::static_pointer_cast(table->schema_), - table->location_carries_paimon_metadata_, options); + return Create(file_system_, root_path_, identifier_, + std::static_pointer_cast(schema_), location_carries_paimon_metadata_, + options); +} + +Result> FormatTable::Copy( + const std::map& dynamic_options) const { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr copied, CopySelf(dynamic_options)); + return std::shared_ptr
(std::move(copied)); } const std::vector& FormatTable::PartitionKeys() const { @@ -233,7 +235,7 @@ Result> FormatTable::NewScan( return Status::Invalid("executor is null pointer"); } PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table, - FormatTable::Copy(SharedSelf(), shared_context->GetOptions())); + CopySelf(shared_context->GetOptions())); // Anything a `ScanContext` carries that a format table cannot honour is refused rather than // silently dropped. if (shared_context->IsStreamingMode()) { @@ -294,7 +296,7 @@ Result> FormatTable::NewRead( return Status::NotImplemented("FullTextMatch predicate execution is not implemented yet"); } PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table, - FormatTable::Copy(SharedSelf(), read_context->GetOptions())); + CopySelf(read_context->GetOptions())); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read, FormatTableRead::Create(table, read_context)); return std::unique_ptr(std::move(read)); @@ -311,8 +313,7 @@ Result> FormatTable::NewWrite( if (context->GetExecutor() == nullptr) { return Status::Invalid("executor is null pointer"); } - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table, - FormatTable::Copy(SharedSelf(), context->GetOptions())); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table, CopySelf(context->GetOptions())); if (context->GetCatalog() != nullptr) { return Status::Invalid( "WithCatalog() requires a native table; use WriteContextBuilder(FormatTable) for a " @@ -354,7 +355,7 @@ Result> FormatTable::NewCommit( return Status::Invalid("executor is null pointer"); } PAIMON_ASSIGN_OR_RAISE(std::shared_ptr given_table, - FormatTable::Copy(SharedSelf(), context->GetOptions())); + CopySelf(context->GetOptions())); if (!context->IgnoreEmptyCommit()) { return Status::NotImplemented( "a format table cannot record an empty commit: keeping one means writing a " diff --git a/src/paimon/core/table/format/format_table_test.cpp b/src/paimon/core/table/format/format_table_test.cpp index 2430ffbda..2dac6c102 100644 --- a/src/paimon/core/table/format/format_table_test.cpp +++ b/src/paimon/core/table/format/format_table_test.cpp @@ -3071,7 +3071,7 @@ TEST(FormatTableTest, TestABranchNamedThroughOptionsIsRefusedAtEveryEntryPoint) ASSERT_OK(WriteAndCommit(table, std::move(batch))); const std::map branch = {{Options::BRANCH, "dev"}}; - ASSERT_NOK_WITH_MSG(FormatTable::Copy(table, branch), "has no branches"); + ASSERT_NOK_WITH_MSG(table->Copy(branch), "has no branches"); ASSERT_NOK_WITH_MSG( FormatTable::Create(dir->GetFileSystem(), dir->Str(), Identifier("db", "tbl"), branch), "has no branches"); @@ -3108,10 +3108,32 @@ TEST(FormatTableTest, TestABranchNamedThroughOptionsIsRefusedAtEveryEntryPoint) // The branch that names no branch at all is the main one, and is left alone: refusing it // would refuse the default every context carries. for (const char* main_branch : {"main", ""}) { - ASSERT_OK(FormatTable::Copy(table, {{Options::BRANCH, main_branch}})) << main_branch; + ASSERT_OK(table->Copy({{Options::BRANCH, main_branch}})) << main_branch; } } +TEST(FormatTableTest, TestATableCopiesItselfWithTheOptionsGivenAtTheCall) { + std::unique_ptr dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + ASSERT_OK_AND_ASSIGN(std::shared_ptr table, + CreateTable(dir->GetFileSystem(), dir->Str(), {"dt"})); + + // The copy overlays what the call gives on the options the schema stored, the precedence + // `Copy()` promises: a probe option lands, `type` - structural - does not. + ASSERT_OK_AND_ASSIGN(std::shared_ptr
copied, + table->Copy({{Options::TYPE, "table"}, {"probe", "on-top"}})); + std::shared_ptr copied_table = std::static_pointer_cast(copied); + EXPECT_EQ(copied_table->Options().at("probe"), "on-top"); + EXPECT_EQ(copied_table->Options().at(Options::TYPE), "format-table"); + EXPECT_EQ(copied_table->FullName(), table->FullName()); + EXPECT_EQ(copied_table->Location(), table->Location()); + EXPECT_EQ(copied_table->GetFileSystem(), table->GetFileSystem()); + + // The refusal `Copy(table, ...)` reports is the one the table itself reports: the copy goes + // through the same entry point, not a shortcut around its checks. + ASSERT_NOK_WITH_MSG(table->Copy({{Options::BRANCH, "dev"}}), "has no branches"); +} + TEST(FormatTableTest, TestAFormatTableSchemaOnABranchIsRefusedRatherThanServedFromTheTableRoot) { // A context that names a path takes its branch from `WithBranch()`, and the schema of that // branch is what the entry point dispatches on. A format table's data is the files under the diff --git a/src/paimon/core/table/source/table_read_test.cpp b/src/paimon/core/table/source/table_read_test.cpp index ead31b17a..3d580d64d 100644 --- a/src/paimon/core/table/source/table_read_test.cpp +++ b/src/paimon/core/table/source/table_read_test.cpp @@ -250,25 +250,41 @@ TEST(TableReadTest, TestCreateAppendOnlyTableRead) { TEST(TableReadTest, TestMergeOptions) { std::string path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09"; - ReadContextBuilder context_builder(path); - context_builder.SetReadFieldNames({"f0", "f1", "f2", "f3"}); - context_builder.AddOption("read.batch-size", "2"); - context_builder.AddOption("orc.read.enable-lazy-decoding", "true"); - context_builder.AddOption("bucket", "10"); - context_builder.AddOption("bucket-key", "f3"); - ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); - auto append_only_table_read = dynamic_cast(table_read.get()); - ASSERT_TRUE(append_only_table_read); - auto* typed_split_read = - dynamic_cast(append_only_table_read->split_reads_[0].get()); - ASSERT_TRUE(typed_split_read); - const auto& core_options = typed_split_read->options_; - std::map expected_options = { - {"read.batch-size", "2"}, {"orc.read.enable-lazy-decoding", "true"}, - {"bucket", "10"}, {"bucket-key", "f3"}, - {"manifest.format", "orc"}, {"file.format", "orc"}}; - ASSERT_EQ(expected_options, core_options.ToMap()); + { + // The options the table was loaded with stand as they are, and a runtime setting the + // context adds on top - the table was loaded with neither of these keys - is served as + // given. + ReadContextBuilder context_builder(path); + context_builder.SetReadFieldNames({"f0", "f1", "f2", "f3"}); + context_builder.AddOption("read.batch-size", "2"); + context_builder.AddOption("orc.read.enable-lazy-decoding", "true"); + ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + auto append_only_table_read = dynamic_cast(table_read.get()); + ASSERT_TRUE(append_only_table_read); + auto* typed_split_read = + dynamic_cast(append_only_table_read->split_reads_[0].get()); + ASSERT_TRUE(typed_split_read); + const auto& core_options = typed_split_read->options_; + std::map expected_options = { + {"read.batch-size", "2"}, + {"orc.read.enable-lazy-decoding", "true"}, + {"bucket", "2"}, + {"bucket-key", "f2"}, + {"manifest.format", "orc"}, + {"file.format", "orc"}}; + ASSERT_EQ(expected_options, core_options.ToMap()); + } + { + // An option that contradicts the table is refused: the read serves the options the table + // was loaded with, and a caller that means to change them copies the table. + ReadContextBuilder context_builder(path); + context_builder.SetReadFieldNames({"f0", "f1", "f2", "f3"}); + context_builder.AddOption("bucket", "10"); + ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); + ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(read_context)), + "option 'bucket' is '10' in the context but '2' on the table"); + } } TEST(TableReadTest, TestDependencyReadWithoutCatalogUsesReadOptions) { diff --git a/src/paimon/core/table/system/system_table.cpp b/src/paimon/core/table/system/system_table.cpp index dd00b7bcd..cf4864071 100644 --- a/src/paimon/core/table/system/system_table.cpp +++ b/src/paimon/core/table/system/system_table.cpp @@ -191,6 +191,11 @@ Status SystemTable::AdoptOwnSchema() { return Status::OK(); } +Result> SystemTable::Copy( + const std::map& /*dynamic_options*/) const { + return Status::NotImplemented(Name(), " system table is read-only: copying is not supported"); +} + Result> SystemTable::NewWrite( std::unique_ptr /*context*/) const { return Status::NotImplemented(Name(), " system table is read-only: writing is not supported"); diff --git a/src/paimon/core/table/system/system_table.h b/src/paimon/core/table/system/system_table.h index 2e4dff55d..96eead153 100644 --- a/src/paimon/core/table/system/system_table.h +++ b/src/paimon/core/table/system/system_table.h @@ -66,6 +66,12 @@ class SystemTable : public Table, public std::enable_shared_from_this> ArrowSchema() const = 0; + /// A system table has no options of its own to override: its schema is the read-only view it + /// exposes, which is not reconfigured by a copy. The data table underneath is copied through + /// its own `Table`. + Result> Copy( + const std::map& dynamic_options) const override; + /// A system table is read-only: writing it or committing for it is refused rather than /// routed to the data table underneath. Result> NewWrite( diff --git a/src/paimon/core/table/system/system_table_test.cpp b/src/paimon/core/table/system/system_table_test.cpp index 49cf6838f..0a9f85a2e 100644 --- a/src/paimon/core/table/system/system_table_test.cpp +++ b/src/paimon/core/table/system/system_table_test.cpp @@ -227,6 +227,20 @@ TEST(SystemTableTest, TestReadOptimizedSystemTableRegistration) { ASSERT_EQ(arrow_schema->field(1)->type()->id(), arrow::Type::INT32); } +TEST(SystemTableTest, TestCopyIsRefused) { + // A system table is a read-only view over the data table underneath it, and the schema it + // exposes is not one a copy reconfigures. The data table is copied through its own `Table`. + std::map options = {{Options::FILE_SYSTEM, "local"}, + {Options::FILE_FORMAT, "orc"}}; + ASSERT_OK_AND_ASSIGN(std::shared_ptr table_schema, + CreateTableSchemaForTest(options)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr system_table, + SystemTableLoader::Load(ReadOptimizedSystemTable::kName, /*fs=*/nullptr, + "/tmp/table", table_schema, + /*dynamic_options=*/{})); + ASSERT_NOK_WITH_MSG(system_table->Copy({{"probe", "on-top"}}), "copying is not supported"); +} + TEST(SystemTableTest, TestReadOptimizedSystemTablePathParsing) { ASSERT_OK_AND_ASSIGN(std::optional parsed, SystemTableLoader::TryParsePath("/tmp/db.db/t$branch_audit$ro")); diff --git a/src/paimon/core/table/table.cpp b/src/paimon/core/table/table.cpp index f9cf78545..cb86e4281 100644 --- a/src/paimon/core/table/table.cpp +++ b/src/paimon/core/table/table.cpp @@ -76,6 +76,7 @@ #include "paimon/core/schema/schema_validation.h" #include "paimon/core/schema/table_schema.h" #include "paimon/core/table/bucket_mode.h" +#include "paimon/core/table/file_store.h" #include "paimon/core/table/file_store_table.h" #include "paimon/core/table/source/abstract_table_scan.h" #include "paimon/core/table/source/append_only_split_generator.h" @@ -217,6 +218,7 @@ Result HasHistoricalPrimaryKeyBTreeDefinition( Result> TableOfLoadedSchema( const std::shared_ptr& file_system, const std::string& path, const std::string& branch, const std::map& options, + const std::shared_ptr& schema_manager, const std::shared_ptr& latest_schema, const std::optional& identifier, const std::string& uuid = "") { PAIMON_ASSIGN_OR_RAISE(std::shared_ptr format_table, @@ -229,7 +231,72 @@ Result> TableOfLoadedSchema( Identifier table_identifier = identifier.value_or(Identifier(Identifier::kUnknownDatabase, PathUtil::GetName(path))); return std::make_shared(file_system, path, latest_schema, table_identifier, - uuid); + uuid, branch, schema_manager); +} + +/// The managed table a context names behind a path: the schema is read through a manager of the +/// context's branch - handed over instead when the caller already read it - and what the schema +/// says is there decides the table `TableOfLoadedSchema` hands back. +/// +/// @param schema_json The schema of the table, when the caller already read it, or null when it +/// has to be read here. It is honoured on the main branch only: a branch publishes its own +/// under `branch/branch-`, which is where the read below goes. +Result> ManagedTableOf(const std::shared_ptr& file_system, + const std::string& path, const std::string& branch, + const std::map& options, + const std::optional& schema_json, + const std::optional& identifier, + const std::string& uuid = "") { + auto schema_manager = std::make_shared(file_system, path, branch); + std::shared_ptr latest_schema; + if (branch == BranchManager::DEFAULT_MAIN_BRANCH && schema_json) { + // Handing the schema over only saves reading it. A context always names a table path and + // this library writes the metadata under it, so `schema` and `branch` stay metadata. + PAIMON_ASSIGN_OR_RAISE(latest_schema, TableSchema::CreateFromJson(schema_json.value())); + } else { + PAIMON_ASSIGN_OR_RAISE(std::optional> latest, + schema_manager->Latest()); + if (latest) { + latest_schema = *latest; + } + } + return TableOfLoadedSchema(file_system, path, branch, options, schema_manager, latest_schema, + identifier, uuid); +} + +/// The identifier a catalog knows the table by, on `branch`: a non-main branch is folded into +/// the identifier, so the catalog is asked for the table on that branch rather than the main one. +Result> MakeBranchCatalogIdentifier( + const std::optional& identifier, const std::string& branch) { + if (!identifier) { + return std::optional(); + } + PAIMON_ASSIGN_OR_RAISE(Identifier branch_identifier, + CatalogUtils::BranchIdentifier(identifier.value(), branch)); + return std::optional(std::move(branch_identifier)); +} + +/// The schema a catalog may hold for the table, read from it on the main branch only and null +/// otherwise. A branch publishes its own schema under `branch/branch-`, which is where +/// `ManagedTableOf` reads instead, so a catalog answering for no branch schema does not fail the +/// `kind` operation to one. `kind` also names the operator the missing-identifier message speaks +/// of. +Result> CatalogMainBranchSchemaJson( + const std::shared_ptr& catalog, const std::optional& catalog_identifier, + const std::string& branch, const char* kind) { + if (catalog == nullptr) { + return std::optional(); + } + if (!catalog_identifier) { + return Status::Invalid(fmt::format("a catalog {} requires a table identifier", kind)); + } + if (BranchManager::IsMainBranch(branch)) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, + catalog->LoadTableSchema(catalog_identifier.value())); + PAIMON_ASSIGN_OR_RAISE(std::string schema_json, schema->GetJsonSchema()); + return std::optional(std::move(schema_json)); + } + return std::optional(); } /// The table a scan context names, read from what the context carries: a format table the @@ -254,22 +321,8 @@ Result> TableOf(const ScanContext& ctx) { // below. One `TableScan` interface serves both, as Java Paimon serves both through one // `ReadBuilder`. const std::string branch = BranchManager::NormalizeBranch(tmp_options.GetBranch()); - std::shared_ptr latest_schema; - if (branch == BranchManager::DEFAULT_MAIN_BRANCH && ctx.GetSpecificTableSchema()) { - // Handing the schema over only saves reading it. A context always names a table path and - // this library writes the metadata under it, so `schema` and `branch` stay metadata. - PAIMON_ASSIGN_OR_RAISE(latest_schema, - TableSchema::CreateFromJson(ctx.GetSpecificTableSchema().value())); - } else { - SchemaManager schema_manager(tmp_options.GetFileSystem(), ctx.GetPath(), branch); - PAIMON_ASSIGN_OR_RAISE(std::optional> latest, - schema_manager.Latest()); - if (latest) { - latest_schema = *latest; - } - } - return TableOfLoadedSchema(tmp_options.GetFileSystem(), ctx.GetPath(), branch, ctx.GetOptions(), - latest_schema, ctx.GetIdentifier()); + return ManagedTableOf(tmp_options.GetFileSystem(), ctx.GetPath(), branch, ctx.GetOptions(), + ctx.GetSpecificTableSchema(), ctx.GetIdentifier()); } /// The table a read context names, read from what the context carries: a format table the @@ -304,22 +357,8 @@ Result> TableOf(const ReadContext& ctx) { // which files there are and how a row is put back together is its own. How a file is opened // is not: both paths go through `DataFileReaderFactory`. const std::string branch = ctx.GetBranch(); - std::shared_ptr latest_schema; - if (branch == BranchManager::DEFAULT_MAIN_BRANCH && ctx.GetSpecificTableSchema()) { - // Handing the schema over only saves reading it. A context always names a table path and - // this library writes the metadata under it, so `schema` and `branch` stay metadata. - PAIMON_ASSIGN_OR_RAISE(latest_schema, - TableSchema::CreateFromJson(ctx.GetSpecificTableSchema().value())); - } else { - SchemaManager schema_manager(tmp_core_options.GetFileSystem(), ctx.GetPath(), branch); - PAIMON_ASSIGN_OR_RAISE(std::optional> latest, - schema_manager.Latest()); - if (latest) { - latest_schema = *latest; - } - } - return TableOfLoadedSchema(tmp_core_options.GetFileSystem(), ctx.GetPath(), branch, - ctx.GetOptions(), latest_schema, ctx.GetIdentifier()); + return ManagedTableOf(tmp_core_options.GetFileSystem(), ctx.GetPath(), branch, ctx.GetOptions(), + ctx.GetSpecificTableSchema(), ctx.GetIdentifier()); } /// The table a write context names, read from what the context carries: a format table the @@ -346,44 +385,18 @@ Result> TableOf(const WriteContext& ctx) { CoreOptions::FromMap(ctx.GetOptions(), specific_fs, ctx.GetFileSystemSchemeToIdentifierMap())); std::string branch = ctx.GetBranch(); - std::optional catalog_identifier; - if (ctx.GetIdentifier()) { - PAIMON_ASSIGN_OR_RAISE(Identifier branch_identifier, - CatalogUtils::BranchIdentifier(ctx.GetIdentifier().value(), branch)); - catalog_identifier.emplace(std::move(branch_identifier)); - } + PAIMON_ASSIGN_OR_RAISE(std::optional catalog_identifier, + MakeBranchCatalogIdentifier(ctx.GetIdentifier(), branch)); // Only the main branch may have a current schema the catalog alone holds. A branch publishes // its own under `branch/branch-`, which is where the load below and a read of the branch // both read it, so a catalog answering for no branch schema does not fail a write to one. - std::optional catalog_table_schema; - if (ctx.GetCatalog() != nullptr) { - if (!catalog_identifier) { - return Status::Invalid("a catalog write requires a table identifier"); - } - if (BranchManager::IsMainBranch(branch)) { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, - ctx.GetCatalog()->LoadTableSchema(catalog_identifier.value())); - PAIMON_ASSIGN_OR_RAISE(catalog_table_schema, schema->GetJsonSchema()); - } - } + PAIMON_ASSIGN_OR_RAISE( + std::optional catalog_table_schema, + CatalogMainBranchSchemaJson(ctx.GetCatalog(), catalog_identifier, branch, "write")); // A format table writes plain data files into a directory, so it never reaches the manifest // path below. - SchemaManager schema_manager(tmp_options.GetFileSystem(), ctx.GetRootPath(), branch); - std::shared_ptr latest_schema; - if (branch == BranchManager::DEFAULT_MAIN_BRANCH && catalog_table_schema) { - // Handing the schema over only saves reading it. A context always names a table path and - // this library writes the metadata under it, so `schema` and `branch` stay metadata. - PAIMON_ASSIGN_OR_RAISE(latest_schema, - TableSchema::CreateFromJson(catalog_table_schema.value())); - } else { - PAIMON_ASSIGN_OR_RAISE(std::optional> latest, - schema_manager.Latest()); - if (latest) { - latest_schema = *latest; - } - } - return TableOfLoadedSchema(tmp_options.GetFileSystem(), ctx.GetRootPath(), branch, - ctx.GetOptions(), latest_schema, ctx.GetIdentifier()); + return ManagedTableOf(tmp_options.GetFileSystem(), ctx.GetRootPath(), branch, ctx.GetOptions(), + catalog_table_schema, ctx.GetIdentifier()); } /// The table a commit context names, read from what the context carries: a format table the @@ -413,47 +426,83 @@ Result> TableOf(const CommitContext& ctx) { PAIMON_ASSIGN_OR_RAISE(std::string branch, BranchManager::ResolveBranch( ctx.GetIdentifier(), ctx.GetOptions(), /*explicit_branch=*/std::nullopt, "commit")); - std::optional catalog_identifier; - if (ctx.GetIdentifier()) { - PAIMON_ASSIGN_OR_RAISE(Identifier branch_identifier, - CatalogUtils::BranchIdentifier(ctx.GetIdentifier().value(), branch)); - catalog_identifier.emplace(std::move(branch_identifier)); - } + PAIMON_ASSIGN_OR_RAISE(std::optional catalog_identifier, + MakeBranchCatalogIdentifier(ctx.GetIdentifier(), branch)); // Catalog-managed schemas may be absent from the table directory, but only on the main branch: // a branch publishes its own under `branch/branch-`, which is where the load below and a // read of the branch both read it, so a catalog answering for no branch schema does not fail // a commit to one. - std::optional catalog_table_schema; - if (ctx.GetCatalog() != nullptr) { - if (!catalog_identifier) { - return Status::Invalid("a catalog commit requires a table identifier"); - } - if (BranchManager::IsMainBranch(branch)) { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr loaded_schema, - ctx.GetCatalog()->LoadTableSchema(catalog_identifier.value())); - PAIMON_ASSIGN_OR_RAISE(std::string schema_json, loaded_schema->GetJsonSchema()); - catalog_table_schema = std::move(schema_json); - } - } + PAIMON_ASSIGN_OR_RAISE( + std::optional catalog_table_schema, + CatalogMainBranchSchemaJson(ctx.GetCatalog(), catalog_identifier, branch, "commit")); // A format table commits by renaming files into place, so it never reaches the snapshot path // below. The managed path here reads the commit's branch, so this reads the same one: the two // must not dispatch on different schemas. - SchemaManager schema_manager(tmp_options.GetFileSystem(), root_path, branch); - std::shared_ptr latest_schema; - if (branch == BranchManager::DEFAULT_MAIN_BRANCH && catalog_table_schema) { - // Handing the schema over only saves reading it. A context always names a table path and - // this library writes the metadata under it, so `schema` and `branch` stay metadata. - PAIMON_ASSIGN_OR_RAISE(latest_schema, - TableSchema::CreateFromJson(catalog_table_schema.value())); - } else { - PAIMON_ASSIGN_OR_RAISE(std::optional> latest, - schema_manager.Latest()); - if (latest) { - latest_schema = *latest; + return ManagedTableOf(tmp_options.GetFileSystem(), root_path, branch, ctx.GetOptions(), + catalog_table_schema, ctx.GetIdentifier(), ctx.GetTableId().value_or("")); +} + +/// The options a context carries against the ones the table was loaded with. A table serves the +/// options it was loaded with, so an option the context contradicts is refused rather than +/// honoured - a caller that means to change them copies the table. A key the table was not loaded +/// with is no contradiction: it names a runtime setting - a scan mode, a batch size - which is +/// none of the table's business, and is served as given. +Status CheckOptionsAgreeWithTable(const TableSchema& table_schema, + const std::map& context_options) { + const std::map& table_options = table_schema.Options(); + for (const auto& [key, value] : context_options) { + auto table_option = table_options.find(key); + if (table_option != table_options.end() && table_option->second != value) { + return Status::Invalid( + fmt::format("option '{}' is '{}' in the context but '{}' on the table", key, value, + table_option->second)); } } - return TableOfLoadedSchema(tmp_options.GetFileSystem(), root_path, branch, ctx.GetOptions(), - latest_schema, ctx.GetIdentifier(), ctx.GetTableId().value_or("")); + return Status::OK(); +} + +/// The checks every operator entry point opens with: a context, a memory pool and an executor +/// must all be there for the operator to be built at all. `kind` names the operator the first +/// message speaks of, which is all that sets the entry points' messages apart. +template +Status CheckOperatorContext(const ContextPointer& context, const char* kind) { + if (context == nullptr) { + return Status::Invalid(fmt::format("{} context is null pointer", kind)); + } + if (context->GetMemoryPool() == nullptr) { + return Status::Invalid("memory pool is null pointer"); + } + if (context->GetExecutor() == nullptr) { + return Status::Invalid("executor is null pointer"); + } + return Status::OK(); +} + +/// The schema the table was constructed with, which a write or a commit serves as the table was +/// loaded rather than reading the schema again. +Result> LatestTableSchemaOfBranch( + const std::shared_ptr& schema, const std::string& branch) { + std::shared_ptr latest_schema = std::static_pointer_cast(schema); + if (latest_schema == nullptr) { + return Status::Invalid(fmt::format("not found latest schema in branch {}", branch)); + } + return latest_schema; +} + +/// The options a write or a commit serves: the ones the table was loaded with - an option the +/// context contradicts is refused by `CheckOptionsAgreeWithTable` above - the ones the context +/// adds on top, and the table's branch, which the metadata paths below are built for: what reads +/// through these options, whose cache keys are scoped by it, has to read the same one. +Result> MergedOperatorOptions( + const TableSchema& table_schema, const std::map& context_options, + const std::string& branch) { + PAIMON_RETURN_NOT_OK(CheckOptionsAgreeWithTable(table_schema, context_options)); + auto opts = table_schema.Options(); + for (const auto& [key, value] : context_options) { + opts[key] = value; + } + opts[Options::BRANCH] = branch; + return opts; } } // namespace @@ -467,60 +516,97 @@ Table::Table(const std::shared_ptr& file_system, const std::string& identifier_(identifier), uuid_(uuid) {} +FileStoreTable::FileStoreTable(const std::shared_ptr& file_system, + const std::string& root_path, const std::shared_ptr& schema, + const Identifier& identifier, const std::string& uuid, + const std::string& branch, + const std::shared_ptr& schema_manager, + const std::shared_ptr& snapshot_manager) + : Table(file_system, root_path, schema, identifier, uuid), + store_(std::make_shared(file_system, root_path, + std::static_pointer_cast(schema), branch, + schema_manager, snapshot_manager)) {} + +Result> FileStoreTable::Copy( + const std::map& dynamic_options) const { + std::shared_ptr loaded_schema = std::static_pointer_cast(schema_); + if (loaded_schema == nullptr) { + return Status::Invalid(fmt::format("not found latest schema of {}", FullName())); + } + // `type` is structural and read from the schema alone, the promise `Copy` makes: an option + // given at the call must not restyle the table. + std::map overridable = dynamic_options; + overridable.erase(Options::TYPE); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr merged_schema, + loaded_schema->MergeOptions(overridable)); + // The managers are handed over, not rebuilt: the copy serves the same root path and branch, + // so its operators read the schema cache this table already warmed. + return std::shared_ptr
(std::make_shared( + file_system_, root_path_, std::shared_ptr(std::move(merged_schema)), + identifier_, CatalogUuid().value_or(""), CurrentBranch(), GetSchemaManager(), + GetSnapshotManager())); +} + +Result> FileStoreTable::SwitchToBranch(const std::string& branch) const { + const std::string target_branch = BranchManager::NormalizeBranch(branch); + if (target_branch == CurrentBranch()) { + // The table already serves this branch: a copy of it - schema and managers shared - is + // the same table to the caller. + return std::shared_ptr
(std::make_shared( + file_system_, root_path_, schema_, identifier_, CatalogUuid().value_or(""), + CurrentBranch(), GetSchemaManager(), GetSnapshotManager())); + } + // The schema of the branch switched to, read from its own directory, which a branch without + // one does not have. + auto schema_manager = std::make_shared(file_system_, root_path_, target_branch); + PAIMON_ASSIGN_OR_RAISE(std::optional> latest, + schema_manager->Latest()); + if (!latest) { + return Status::Invalid(fmt::format("branch {} does not exist", target_branch)); + } + // A branch other than main is named in the options the schema carries - the way a read + // context names it - so what reads through them serves the branch the managers point at. + std::map overridable; + if (target_branch != BranchManager::DEFAULT_MAIN_BRANCH) { + overridable[Options::BRANCH] = target_branch; + } + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr branch_schema, + (*latest)->MergeOptions(overridable)); + return std::shared_ptr
(std::make_shared( + file_system_, root_path_, std::shared_ptr(std::move(branch_schema)), + identifier_, CatalogUuid().value_or(""), target_branch, schema_manager)); +} + Result> FileStoreTable::NewWrite( std::unique_ptr ctx) const { - if (ctx == nullptr) { - return Status::Invalid("write context is null pointer"); - } - if (ctx->GetMemoryPool() == nullptr) { - return Status::Invalid("memory pool is null pointer"); - } - if (ctx->GetExecutor() == nullptr) { - return Status::Invalid("executor is null pointer"); - } + PAIMON_RETURN_NOT_OK(CheckOperatorContext(ctx, "write")); PAIMON_RETURN_NOT_OK(CheckVersionManagementImplemented(ctx->GetCatalog())); std::shared_ptr specific_fs = ctx->GetSpecificFileSystem(); + // A catalog write names its table; the file system itself comes from the table, which its + // catalog's credentials already opened. if (specific_fs == nullptr && ctx->GetFileSystemSchemeToIdentifierMap().empty() && - ctx->GetCatalog() != nullptr) { - if (!ctx->GetIdentifier()) { - return Status::Invalid("a catalog write requires a table identifier"); - } - // A catalog issuing per-table temporary credentials only hands them out through - // GetTableFileSystem, so the write uses the credentials of this table. - PAIMON_ASSIGN_OR_RAISE(specific_fs, - ctx->GetCatalog()->GetTableFileSystem(ctx->GetIdentifier().value())); + ctx->GetCatalog() != nullptr && !ctx->GetIdentifier()) { + return Status::Invalid("a catalog write requires a table identifier"); } - PAIMON_ASSIGN_OR_RAISE(CoreOptions tmp_options, - CoreOptions::FromMap(ctx->GetOptions(), specific_fs, - ctx->GetFileSystemSchemeToIdentifierMap())); - std::string branch = ctx->GetBranch(); - std::optional catalog_identifier; - if (ctx->GetIdentifier()) { - PAIMON_ASSIGN_OR_RAISE( - Identifier branch_identifier, - CatalogUtils::BranchIdentifier(ctx->GetIdentifier().value(), branch)); - catalog_identifier.emplace(std::move(branch_identifier)); - } - auto schema_manager = - std::make_shared(tmp_options.GetFileSystem(), ctx->GetRootPath(), branch); + // The write goes to the branch the table serves: a table is the branch it was loaded for, + // so the context's own branch is not consulted. + PAIMON_ASSIGN_OR_RAISE(std::optional catalog_identifier, + MakeBranchCatalogIdentifier(ctx->GetIdentifier(), CurrentBranch())); // The schema the table was constructed with, so a write serves the table as it was loaded // rather than reading the schema again. - std::shared_ptr latest_schema = std::static_pointer_cast(schema_); - if (latest_schema == nullptr) { - return Status::Invalid(fmt::format("not found latest schema in branch {}", branch)); - } - const std::shared_ptr& schema = latest_schema; + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, + LatestTableSchemaOfBranch(schema_, CurrentBranch())); auto arrow_schema = DataField::ConvertDataFieldsToArrowSchema(schema->Fields()); - auto opts = schema->Options(); - for (const auto& [key, value] : ctx->GetOptions()) { - opts[key] = value; - } - // The metadata paths above are built for this branch, so what reads through these options, - // whose cache keys are scoped by it, has to read the same one. - opts[Options::BRANCH] = branch; + // The options the table was loaded with, with a runtime setting the context adds on top: an + // option the context would have changed is refused rather than honoured, and the branch is + // the table's - the metadata paths below are built for it, so what reads through these + // options, whose cache keys are scoped by it, has to read the same one. + std::map opts; + PAIMON_ASSIGN_OR_RAISE(opts, + MergedOperatorOptions(*schema, ctx->GetOptions(), CurrentBranch())); PAIMON_ASSIGN_OR_RAISE( CoreOptions options, - CoreOptions::FromMap(opts, specific_fs, ctx->GetFileSystemSchemeToIdentifierMap())); + CoreOptions::FromMap(opts, file_system_, ctx->GetFileSystemSchemeToIdentifierMap())); PAIMON_RETURN_NOT_OK(SchemaValidation::ValidateRealtime(*schema, options)); PAIMON_RETURN_NOT_OK(SchemaValidation::ValidateVectorStore(*schema, options)); if (options.GetVectorFileFormat() && !ctx->GetWriteSchema().empty()) { @@ -530,29 +616,14 @@ Result> FileStoreTable::NewWrite( PAIMON_ASSIGN_OR_RAISE(std::shared_ptr partition_schema, FieldMapping::GetPartitionSchema(arrow_schema, schema->PartitionKeys())); - PAIMON_ASSIGN_OR_RAISE(std::vector external_paths, options.CreateExternalPaths()); - PAIMON_ASSIGN_OR_RAISE(std::optional global_index_external_path, - options.CreateGlobalIndexExternalPath()); - PAIMON_ASSIGN_OR_RAISE( std::shared_ptr file_store_path_factory, - FileStorePathFactory::Create(ctx->GetRootPath(), arrow_schema, schema->PartitionKeys(), - options.GetPartitionDefaultName(), - options.GetWriteFileFormat(/*level=*/0)->Identifier(), - options.DataFilePrefix(), options.LegacyPartitionNameEnabled(), - external_paths, global_index_external_path, - options.IndexFileInDataFileDir(), ctx->GetMemoryPool())); - auto snapshot_manager = - std::make_shared(options.GetFileSystem(), ctx->GetRootPath(), branch); - if (VersionManagedCatalog* versioned = AsVersionManaged(ctx->GetCatalog())) { - std::shared_ptr catalog = ctx->GetCatalog(); - Identifier identifier = catalog_identifier.value(); - // Keep the catalog alive while the writer uses its snapshot loader. - snapshot_manager->SetSnapshotLoader( - [catalog, versioned, identifier]() -> Result> { - return versioned->LoadSnapshot(identifier); - }); - } + store_->NewPathFactory(arrow_schema, options, + options.GetWriteFileFormat(/*level=*/0)->Identifier(), + ctx->GetMemoryPool())); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr snapshot_manager, + store_->NewSnapshotManager(options, ctx->GetCatalog(), catalog_identifier)); std::shared_ptr io_manager; const auto& io_temp_dir = ctx->GetTempDirectory(); if (!io_temp_dir.empty()) { @@ -632,8 +703,8 @@ Result> FileStoreTable::NewWrite( } auto file_store_write = std::make_unique( - file_store_path_factory, snapshot_manager, schema_manager, ctx->GetCommitUser(), - ctx->GetRootPath(), schema, arrow_schema, write_schema, realtime_schema_layout, + file_store_path_factory, snapshot_manager, GetSchemaManager(), ctx->GetCommitUser(), + root_path_, schema, arrow_schema, write_schema, realtime_schema_layout, partition_schema, dv_maintainer_factory, io_manager, options, ignore_previous_files, ctx->IsStreamingMode(), ctx->IgnoreNumBucketCheck(), ctx->GetRealtimeContext(), ctx->GetExecutor(), ctx->GetMemoryPool()); @@ -662,11 +733,11 @@ Result> FileStoreTable::NewWrite( } if (options.GetBucket() == BucketModeDefine::POSTPONE_BUCKET) { return PostponeBucketFileStoreWrite::Create( - snapshot_manager, schema_manager, ctx->GetCommitUser(), ctx->GetRootPath(), schema, + snapshot_manager, GetSchemaManager(), ctx->GetCommitUser(), root_path_, schema, arrow_schema, partition_schema, io_manager, options, ctx->IsStreamingMode(), ctx->IgnoreNumBucketCheck(), ctx->GetWriteId(), ctx->GetFileSystemSchemeToIdentifierMap(), ctx->GetExecutor(), ctx->GetMemoryPool(), - specific_fs); + file_system_); } if (options.GetBucket() <= 0) { return Status::Invalid( @@ -710,7 +781,7 @@ Result> FileStoreTable::NewWrite( if (!has_btree_index) { PAIMON_ASSIGN_OR_RAISE( bool has_historical_btree_index, - HasHistoricalPrimaryKeyBTreeDefinition(schema_manager, schema->Id())); + HasHistoricalPrimaryKeyBTreeDefinition(GetSchemaManager(), schema->Id())); if (has_historical_btree_index) { PAIMON_ASSIGN_OR_RAISE(latest_snapshot, snapshot_manager->LatestSnapshot()); has_existing_index_manifest = @@ -749,9 +820,10 @@ Result> FileStoreTable::NewWrite( if (has_btree_index || has_restored_btree_index) { primary_key_index_maintainer_factory = std::make_shared( - ctx->GetRootPath(), branch, schema, primary_key_index_definitions.Definitions(), - file_store_path_factory, index_file_handler, options, io_manager, - ctx->EnableMultiThreadSpill(), ctx->GetExecutor(), ctx->GetMemoryPool()); + root_path_, CurrentBranch(), schema, + primary_key_index_definitions.Definitions(), file_store_path_factory, + index_file_handler, options, io_manager, ctx->EnableMultiThreadSpill(), + ctx->GetExecutor(), ctx->GetMemoryPool()); } std::shared_ptr realtime_schema_layout; @@ -763,8 +835,8 @@ Result> FileStoreTable::NewWrite( } return std::make_unique( - file_store_path_factory, snapshot_manager, schema_manager, ctx->GetCommitUser(), - ctx->GetRootPath(), schema, arrow_schema, realtime_schema_layout, partition_schema, + file_store_path_factory, snapshot_manager, GetSchemaManager(), ctx->GetCommitUser(), + root_path_, schema, arrow_schema, realtime_schema_layout, partition_schema, dv_maintainer_factory, primary_key_index_maintainer_factory, io_manager, key_comparator, sequence_fields_comparator, merge_function_wrapper, options, ignore_previous_files, ctx->IsStreamingMode(), ctx->IgnoreNumBucketCheck(), ctx->EnableMultiThreadSpill(), @@ -773,15 +845,7 @@ Result> FileStoreTable::NewWrite( } Result> FileStoreWrite::Create(std::unique_ptr ctx) { - if (ctx == nullptr) { - return Status::Invalid("write context is null pointer"); - } - if (ctx->GetMemoryPool() == nullptr) { - return Status::Invalid("memory pool is null pointer"); - } - if (ctx->GetExecutor() == nullptr) { - return Status::Invalid("executor is null pointer"); - } + PAIMON_RETURN_NOT_OK(CheckOperatorContext(ctx, "write")); // First the table the context names, then the context handed to that table's `NewWrite`. PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, TableOf(*ctx)); return table->NewWrite(std::move(ctx)); @@ -791,18 +855,19 @@ namespace { class TableScanImpl { public: + /// @param snapshot_manager The snapshot manager the scan reads through - the table's own. + /// @param schema_manager The schema manager the scan reads through, likewise. static Result> CreateFileStoreScan( const std::shared_ptr& path_factory, const std::shared_ptr& arrow_schema, const std::shared_ptr& table_schema, const CoreOptions& core_options, + const std::shared_ptr& snapshot_manager, + const std::shared_ptr& schema_manager, const std::shared_ptr& executor, const std::shared_ptr& memory_pool, - const ScanContext* context) { + const ScanContext* context, const std::string& root_path) { auto fs = core_options.GetFileSystem(); auto manifest_file_format = core_options.GetManifestFormat(); - std::string branch = BranchManager::NormalizeBranch(core_options.GetBranch()); - auto snapshot_manager = std::make_shared(fs, context->GetPath(), branch); // TODO(liancheng.lsz): support fallback branch in scan - auto schema_manager = std::make_shared(fs, context->GetPath(), branch); PAIMON_ASSIGN_OR_RAISE( std::shared_ptr manifest_list, ManifestList::Create(fs, manifest_file_format, core_options.GetManifestCompression(), @@ -837,12 +902,12 @@ class TableScanImpl { context->GetScanFilters(), core_options, executor, memory_pool)); } - return WithTablePath(std::move(scan), context); + return WithTablePath(std::move(scan), root_path); } static std::unique_ptr WithTablePath(std::unique_ptr&& scan, - const ScanContext* context) { - scan->WithTablePath(context->GetPath()); + const std::string& root_path) { + scan->WithTablePath(root_path); return std::move(scan); } @@ -938,15 +1003,7 @@ Status ValidateRealtimeScan(const TableSchema& table_schema, const CoreOptions& Result> FileStoreTable::NewScan( std::unique_ptr context) const { - if (context == nullptr) { - return Status::Invalid("scan context is null pointer"); - } - if (context->GetMemoryPool() == nullptr) { - return Status::Invalid("memory pool is null pointer"); - } - if (context->GetExecutor() == nullptr) { - return Status::Invalid("executor is null pointer"); - } + PAIMON_RETURN_NOT_OK(CheckOperatorContext(context, "scan")); // The schema the table was constructed with, so a scan serves the table as it was loaded // rather than reading the schema again. std::shared_ptr loaded_schema = std::static_pointer_cast(schema_); @@ -954,16 +1011,29 @@ Result> FileStoreTable::NewScan( return Status::Invalid("not found latest schema"); } const std::shared_ptr& table_schema = loaded_schema; + PAIMON_RETURN_NOT_OK(CheckOptionsAgreeWithTable(*table_schema, context->GetOptions())); PAIMON_ASSIGN_OR_RAISE(bool read_optimized, OptionsUtils::GetValueFromMap(context->GetOptions(), kReadOptimizedScanOption, false)); - // merge options + // The options the table was loaded with, with a runtime setting the context adds on top: an + // option the context would have changed is refused above, so the table's own stand as they + // are. The branch is the table's, not the context's: a table is the branch it was loaded + // for, so a branch the context carries is not honoured. auto options = table_schema->Options(); for (const auto& [key, value] : context->GetOptions()) { - options[key] = value; + if (key != Options::BRANCH) { + options[key] = value; + } + } + if (BranchManager::IsMainBranch(CurrentBranch())) { + options.erase(Options::BRANCH); + } else { + options[Options::BRANCH] = CurrentBranch(); } + // The file system and root path the table was opened with, so a scan serves the table as it + // was loaded rather than resolving them from the context. PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, - CoreOptions::FromMap(options, context->GetSpecificFileSystem(), {})); + CoreOptions::FromMap(options, file_system_, {})); core_options.WithCache(context->GetCache()); PAIMON_RETURN_NOT_OK(SchemaValidation::ValidateRealtime(*table_schema, core_options)); @@ -989,24 +1059,17 @@ Result> FileStoreTable::NewScan( *arrow_schema, context->GetScanFilters()->GetPredicate(), /*validate_field_idx=*/false)); } - PAIMON_ASSIGN_OR_RAISE(std::vector external_paths, - core_options.CreateExternalPaths()); - PAIMON_ASSIGN_OR_RAISE(std::optional global_index_external_path, - core_options.CreateGlobalIndexExternalPath()); - - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr path_factory, - FileStorePathFactory::Create( - context->GetPath(), arrow_schema, table_schema->PartitionKeys(), - core_options.GetPartitionDefaultName(), core_options.GetFileFormat()->Identifier(), - core_options.DataFilePrefix(), core_options.LegacyPartitionNameEnabled(), - external_paths, global_index_external_path, core_options.IndexFileInDataFileDir(), - context->GetMemoryPool())); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr path_factory, + store_->NewPathFactory(arrow_schema, core_options, + core_options.GetFileFormat()->Identifier(), + context->GetMemoryPool())); + // The managers the table carries, whose schema cache the dispatch already warmed. PAIMON_ASSIGN_OR_RAISE(std::shared_ptr file_store_scan, TableScanImpl::CreateFileStoreScan( path_factory, arrow_schema, table_schema, core_options, - context->GetExecutor(), context->GetMemoryPool(), context.get())); + GetSnapshotManager(), GetSchemaManager(), context->GetExecutor(), + context->GetMemoryPool(), context.get(), root_path_)); PAIMON_ASSIGN_OR_RAISE( std::unique_ptr split_generator, TableScanImpl::CreateSplitGenerator(table_schema, core_options, context.get())); @@ -1036,7 +1099,7 @@ Result> FileStoreTable::NewScan( } if (core_options.DataEvolutionEnabled()) { return std::make_unique( - context->GetPath(), snapshot_reader, std::move(batch_scan), table_schema, + root_path_, snapshot_reader, std::move(batch_scan), table_schema, context->GetGlobalIndexResult(), core_options, context->GetMemoryPool(), context->GetExecutor()); } @@ -1053,15 +1116,7 @@ Result> FileStoreTable::NewScan( } Result> TableScan::Create(std::unique_ptr context) { - if (context == nullptr) { - return Status::Invalid("scan context is null pointer"); - } - if (context->GetMemoryPool() == nullptr) { - return Status::Invalid("memory pool is null pointer"); - } - if (context->GetExecutor() == nullptr) { - return Status::Invalid("executor is null pointer"); - } + PAIMON_RETURN_NOT_OK(CheckOperatorContext(context, "scan")); // First the table the context names, then the context handed to that table's `NewScan`. PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, TableOf(*context)); return table->NewScan(std::move(context)); @@ -1069,13 +1124,20 @@ Result> TableScan::Create(std::unique_ptr> CreateInternalReadContext( const std::shared_ptr& context, const std::string& branch, - const std::shared_ptr& loaded_schema) { - std::map tmp_options = context->GetOptions(); + const std::shared_ptr& schema_manager, + const std::shared_ptr& loaded_schema, + const std::shared_ptr& file_system, const std::string& root_path) { std::shared_ptr table_schema; const auto& specific_table_schema = context->GetSpecificTableSchema(); if (loaded_schema != nullptr) { @@ -1084,29 +1146,29 @@ Result> CreateInternalReadContext( PAIMON_ASSIGN_OR_RAISE(table_schema, TableSchema::CreateFromJson(specific_table_schema.value())); } else { - PAIMON_ASSIGN_OR_RAISE(CoreOptions tmp_core_options, - CoreOptions::FromMap(tmp_options, context->GetSpecificFileSystem(), - context->GetFileSystemSchemeToIdentifierMap())); - SchemaManager schema_manager(tmp_core_options.GetFileSystem(), context->GetPath(), branch); PAIMON_ASSIGN_OR_RAISE(std::optional> latest_schema, - schema_manager.Latest()); + schema_manager->Latest()); if (!latest_schema) { - return Status::Invalid(fmt::format("schema file not found in path {}, branch {}", - context->GetPath(), branch)); + return Status::Invalid( + fmt::format("schema file not found in path {}, branch {}", root_path, branch)); } table_schema = latest_schema.value(); } assert(table_schema); - // merge options + // The options the schema was loaded with, with a runtime setting the context adds on top - + // and the fallback below serves the ones its own schema was loaded with. The branch is the + // one named below, not the context's: a branch the context carries is not honoured here. auto options = table_schema->Options(); - for (const auto& [key, value] : tmp_options) { - options[key] = value; + for (const auto& [key, value] : context->GetOptions()) { + if (key != Options::BRANCH) { + options[key] = value; + } } if (branch != BranchManager::DEFAULT_MAIN_BRANCH) { options[Options::BRANCH] = branch; } - return InternalReadContext::Create(context, table_schema, options); + return InternalReadContext::Create(context, table_schema, options, file_system, &root_path); } Result> CreateTableRead( @@ -1125,19 +1187,11 @@ Result> CreateTableRead( } PAIMON_RETURN_NOT_OK(SchemaValidation::ValidateRealtime(*table_schema, core_options)); auto arrow_schema = DataField::ConvertDataFieldsToArrowSchema(table_schema->Fields()); - PAIMON_ASSIGN_OR_RAISE(std::vector external_paths, - core_options.CreateExternalPaths()); - PAIMON_ASSIGN_OR_RAISE(std::optional global_index_external_path, - core_options.CreateGlobalIndexExternalPath()); - PAIMON_ASSIGN_OR_RAISE( std::shared_ptr path_factory, - FileStorePathFactory::Create( - internal_context->GetPath(), arrow_schema, table_schema->PartitionKeys(), - core_options.GetPartitionDefaultName(), core_options.GetFileFormat()->Identifier(), - core_options.DataFilePrefix(), core_options.LegacyPartitionNameEnabled(), - external_paths, global_index_external_path, core_options.IndexFileInDataFileDir(), - memory_pool)); + FileStore::MakePathFactory(internal_context->GetPath(), arrow_schema, + table_schema->PartitionKeys(), core_options, + core_options.GetFileFormat()->Identifier(), memory_pool)); if (internal_context->GetPrimaryKeys().empty()) { return std::make_unique(path_factory, internal_context, memory_pool, @@ -1151,21 +1205,21 @@ Result> CreateTableRead( Result> FileStoreTable::NewRead( std::unique_ptr context) const { std::shared_ptr read_context = std::move(context); - if (read_context == nullptr) { - return Status::Invalid("read context is null pointer"); - } - if (read_context->GetMemoryPool() == nullptr) { - return Status::Invalid("memory pool is null pointer"); - } - if (read_context->GetExecutor() == nullptr) { - return Status::Invalid("executor is null pointer"); - } + PAIMON_RETURN_NOT_OK(CheckOperatorContext(read_context, "read")); // The schema the table was constructed with, so a read serves the table as it was loaded - // rather than reading the schema again. + // rather than reading the schema again. Its options are the ones the read serves: an option + // the context would have changed is refused rather than honoured. std::shared_ptr loaded_schema = std::static_pointer_cast(schema_); + if (loaded_schema == nullptr) { + return Status::Invalid("not found latest schema"); + } + PAIMON_RETURN_NOT_OK(CheckOptionsAgreeWithTable(*loaded_schema, read_context->GetOptions())); + // The branch the table serves, which the read reads on: a table is the branch it was loaded + // for, so the context's own branch is not consulted. PAIMON_ASSIGN_OR_RAISE( std::shared_ptr internal_context, - CreateInternalReadContext(read_context, read_context->GetBranch(), loaded_schema)); + CreateInternalReadContext(read_context, CurrentBranch(), GetSchemaManager(), loaded_schema, + file_system_, root_path_)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_read, CreateTableRead(internal_context, read_context->GetMemoryPool(), @@ -1178,10 +1232,14 @@ Result> FileStoreTable::NewRead( return std::move(table_read); } + // The fallback branch is not the table's, so its schema is read through a manager of its + // own rather than the table's. PAIMON_ASSIGN_OR_RAISE( std::shared_ptr fallback_context, - CreateInternalReadContext(read_context, /*branch=*/scan_fallback_branch.value(), - /*loaded_schema=*/nullptr)); + CreateInternalReadContext( + read_context, /*branch=*/scan_fallback_branch.value(), + std::make_shared(file_system_, root_path_, scan_fallback_branch.value()), + /*loaded_schema=*/nullptr, file_system_, root_path_)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr fallback_table_read, CreateTableRead(fallback_context, read_context->GetMemoryPool(), @@ -1192,15 +1250,7 @@ Result> FileStoreTable::NewRead( } Result> TableRead::Create(std::unique_ptr ctx) { - if (ctx == nullptr) { - return Status::Invalid("read context is null pointer"); - } - if (ctx->GetMemoryPool() == nullptr) { - return Status::Invalid("memory pool is null pointer"); - } - if (ctx->GetExecutor() == nullptr) { - return Status::Invalid("executor is null pointer"); - } + PAIMON_RETURN_NOT_OK(CheckOperatorContext(ctx, "read")); // First the table the context names, then the context handed to that table's `NewRead`. PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, TableOf(*ctx)); return table->NewRead(std::move(ctx)); @@ -1283,41 +1333,19 @@ Result> NewSnapshotCommit( Result> FileStoreTable::NewCommit( std::unique_ptr ctx) const { - if (ctx == nullptr) { - return Status::Invalid("commit context is null pointer"); - } - if (ctx->GetMemoryPool() == nullptr) { - return Status::Invalid("memory pool is null pointer"); - } - if (ctx->GetExecutor() == nullptr) { - return Status::Invalid("executor is null pointer"); - } + PAIMON_RETURN_NOT_OK(CheckOperatorContext(ctx, "commit")); PAIMON_RETURN_NOT_OK(CheckVersionManagementImplemented(ctx->GetCatalog())); - // Use catalog credentials unless the caller supplied a file system. std::shared_ptr specific_fs = ctx->GetSpecificFileSystem(); - if (specific_fs == nullptr && ctx->GetCatalog() != nullptr) { - if (!ctx->GetIdentifier()) { - return Status::Invalid("a catalog commit requires a table identifier"); - } - // Use the credentials of this table, which a catalog issuing per-table temporary - // ones only hands out through GetTableFileSystem. - PAIMON_ASSIGN_OR_RAISE(specific_fs, - ctx->GetCatalog()->GetTableFileSystem(ctx->GetIdentifier().value())); - } - PAIMON_ASSIGN_OR_RAISE(CoreOptions tmp_options, - CoreOptions::FromMap(ctx->GetOptions(), specific_fs)); - const std::string& root_path = ctx->GetRootPath(); - // The branch the context was built for, already validated by `CommitContextBuilder`. - PAIMON_ASSIGN_OR_RAISE(std::string branch, BranchManager::ResolveBranch( - ctx->GetIdentifier(), ctx->GetOptions(), - /*explicit_branch=*/std::nullopt, "commit")); - std::optional catalog_identifier; - if (ctx->GetIdentifier()) { - PAIMON_ASSIGN_OR_RAISE( - Identifier branch_identifier, - CatalogUtils::BranchIdentifier(ctx->GetIdentifier().value(), branch)); - catalog_identifier.emplace(std::move(branch_identifier)); - } + // A catalog commit names its table; the file system itself comes from the table, which its + // catalog's credentials already opened. + if (specific_fs == nullptr && ctx->GetCatalog() != nullptr && !ctx->GetIdentifier()) { + return Status::Invalid("a catalog commit requires a table identifier"); + } + // A commit commits to the root path the table was opened with and the branch the table + // serves, in place of the ones named by the context: a table is the branch it was loaded + // for, so the context's own branch is not consulted. + PAIMON_ASSIGN_OR_RAISE(std::optional catalog_identifier, + MakeBranchCatalogIdentifier(ctx->GetIdentifier(), CurrentBranch())); // The id a snapshot records the catalog's current schema by, read through the loader below // rather than trusted from a schema this commit will not re-read. FileStoreCommitImpl::SchemaIdLoader schema_id_loader; @@ -1339,33 +1367,30 @@ Result> FileStoreTable::NewCommit( return data_schema->Id(); }; } - auto schema_manager = - std::make_shared(tmp_options.GetFileSystem(), root_path, branch); // The schema the table was constructed with, so a commit serves the table as it was loaded // rather than reading the schema again. - std::shared_ptr latest_schema = std::static_pointer_cast(schema_); - if (latest_schema == nullptr) { - return Status::Invalid(fmt::format("not found latest schema in branch {}", branch)); - } - const std::shared_ptr& schema = latest_schema; - auto opts = schema->Options(); - for (const auto& [key, value] : ctx->GetOptions()) { - opts[key] = value; - } - // The metadata paths below are built for this branch, so what reads through these options, - // whose cache keys are scoped by it, has to read the same one. - opts[Options::BRANCH] = branch; + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, + LatestTableSchemaOfBranch(schema_, CurrentBranch())); std::shared_ptr arrow_schema = DataField::ConvertDataFieldsToArrowSchema(schema->Fields()); - PAIMON_ASSIGN_OR_RAISE(CoreOptions options, CoreOptions::FromMap(opts, specific_fs)); + // The options the table was loaded with, with a runtime setting the context adds on top: an + // option the context would have changed is refused rather than honoured, and the branch is + // the table's - the metadata paths below are built for it, so what reads through these + // options, whose cache keys are scoped by it, has to read the same one. + std::map opts; + PAIMON_ASSIGN_OR_RAISE(opts, + MergedOperatorOptions(*schema, ctx->GetOptions(), CurrentBranch())); + PAIMON_ASSIGN_OR_RAISE(CoreOptions options, CoreOptions::FromMap(opts, file_system_)); PAIMON_RETURN_NOT_OK(SchemaValidation::ValidateRealtime(*schema, options)); assert(options.GetFileSystem()); assert(options.GetFileFormat()); PAIMON_RETURN_NOT_OK(FileStoreCommitImpl::ValidateCommitOptions(options)); - PAIMON_ASSIGN_OR_RAISE(bool is_object_store, FileSystem::IsObjectStore(root_path)); - // Catalog commits do not require atomic rename on the object store. + PAIMON_ASSIGN_OR_RAISE(bool is_object_store, FileSystem::IsObjectStore(root_path_)); + // Catalog commits do not require atomic rename on the object store. The inte-test marker is + // not a table option, so it is read from the context the commit was handed. if (is_object_store && !ctx->UseRESTCatalogCommit() && !IsCatalogCommit(*ctx) && - opts.find("enable-object-store-commit-in-inte-test") == opts.end()) { + ctx->GetOptions().find("enable-object-store-commit-in-inte-test") == + ctx->GetOptions().end()) { return Status::NotImplemented( "commit operation does not support object store file system for now"); } @@ -1374,30 +1399,14 @@ Result> FileStoreTable::NewCommit( options.GetPartitionDefaultName(), options.LegacyPartitionNameEnabled(), ctx->GetMemoryPool())); - PAIMON_ASSIGN_OR_RAISE(std::vector external_paths, options.CreateExternalPaths()); - PAIMON_ASSIGN_OR_RAISE(std::optional global_index_external_path, - options.CreateGlobalIndexExternalPath()); - PAIMON_ASSIGN_OR_RAISE( std::shared_ptr path_factory, - FileStorePathFactory::Create( - root_path, arrow_schema, schema->PartitionKeys(), options.GetPartitionDefaultName(), - options.GetFileFormat()->Identifier(), options.DataFilePrefix(), - options.LegacyPartitionNameEnabled(), external_paths, global_index_external_path, - options.IndexFileInDataFileDir(), ctx->GetMemoryPool())); - - auto snapshot_manager = - std::make_shared(options.GetFileSystem(), root_path, branch); - if (IsCatalogCommit(*ctx)) { - std::shared_ptr catalog = ctx->GetCatalog(); - VersionManagedCatalog* versioned = AsVersionManaged(catalog); - Identifier identifier = catalog_identifier.value(); - // Capture the catalog to keep the versioned interface alive. - snapshot_manager->SetSnapshotLoader( - [catalog, versioned, identifier]() -> Result> { - return versioned->LoadSnapshot(identifier); - }); - } + store_->NewPathFactory(arrow_schema, options, options.GetFileFormat()->Identifier(), + ctx->GetMemoryPool())); + + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr snapshot_manager, + store_->NewSnapshotManager(options, ctx->GetCatalog(), catalog_identifier)); PAIMON_ASSIGN_OR_RAISE( std::shared_ptr manifest_list, ManifestList::Create(options.GetFileSystem(), options.GetManifestFormat(), @@ -1425,11 +1434,11 @@ Result> FileStoreTable::NewCommit( CommitScanner::ScanSupplier scan_supplier; if (schema->PrimaryKeys().empty()) { - scan_supplier = CreateAppendScanSupplier(snapshot_manager, schema_manager, manifest_list, - manifest_file, schema, arrow_schema, options, - ctx->GetExecutor(), ctx->GetMemoryPool()); + scan_supplier = CreateAppendScanSupplier(snapshot_manager, GetSchemaManager(), + manifest_list, manifest_file, schema, arrow_schema, + options, ctx->GetExecutor(), ctx->GetMemoryPool()); } else { - scan_supplier = CreatePkScanSupplier(snapshot_manager, schema_manager, manifest_list, + scan_supplier = CreatePkScanSupplier(snapshot_manager, GetSchemaManager(), manifest_list, manifest_file, schema, arrow_schema, options, ctx->GetExecutor(), ctx->GetMemoryPool()); } @@ -1439,24 +1448,16 @@ Result> FileStoreTable::NewCommit( NewSnapshotCommit(*ctx, catalog_identifier, options.GetFileSystem(), snapshot_manager)); return std::make_unique( - ctx->GetMemoryPool(), ctx->GetExecutor(), arrow_schema, root_path, ctx->GetCommitUser(), + ctx->GetMemoryPool(), ctx->GetExecutor(), arrow_schema, root_path_, ctx->GetCommitUser(), options, path_factory, std::move(partition_computer), snapshot_manager, snapshot_commit, ctx->IgnoreEmptyCommit(), ctx->AppendCommitCheckConflict(), schema, std::move(schema_id_loader), manifest_file, manifest_list, index_manifest_file, - expire_snapshots, schema_manager, std::move(scan_supplier)); + expire_snapshots, GetSchemaManager(), std::move(scan_supplier)); } Result> FileStoreCommit::Create( std::unique_ptr ctx) { - if (ctx == nullptr) { - return Status::Invalid("commit context is null pointer"); - } - if (ctx->GetMemoryPool() == nullptr) { - return Status::Invalid("memory pool is null pointer"); - } - if (ctx->GetExecutor() == nullptr) { - return Status::Invalid("executor is null pointer"); - } + PAIMON_RETURN_NOT_OK(CheckOperatorContext(ctx, "commit")); // First the table the context names, then the context handed to that table's `NewCommit`. PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, TableOf(*ctx)); return table->NewCommit(std::move(ctx)); diff --git a/src/paimon/core/table/table_test.cpp b/src/paimon/core/table/table_test.cpp index 1ad8d9cd3..01a7ce27e 100644 --- a/src/paimon/core/table/table_test.cpp +++ b/src/paimon/core/table/table_test.cpp @@ -25,22 +25,29 @@ #include #include "arrow/api.h" +#include "arrow/c/abi.h" +#include "arrow/c/bridge.h" #include "gtest/gtest.h" #include "paimon/commit_context.h" +#include "paimon/commit_message.h" #include "paimon/common/utils/checked_cast.h" #include "paimon/core/catalog/file_system_catalog.h" #include "paimon/core/schema/schema_manager.h" #include "paimon/core/schema/table_schema.h" #include "paimon/core/table/file_store_table.h" +#include "paimon/defs.h" #include "paimon/file_store_commit.h" #include "paimon/file_store_write.h" #include "paimon/read_context.h" +#include "paimon/record_batch.h" #include "paimon/scan_context.h" #include "paimon/schema/schema.h" #include "paimon/table/format/format_table.h" +#include "paimon/table/source/plan.h" #include "paimon/table/source/table_read.h" #include "paimon/table/source/table_scan.h" #include "paimon/testing/mock/mock_catalog.h" +#include "paimon/testing/mock/mock_file_system.h" #include "paimon/testing/utils/testharness.h" #include "paimon/write_context.h" @@ -49,23 +56,198 @@ namespace paimon::test { namespace { // Creates a managed table directory and a `Table` over it, holding the schema written to -// `branch`'s directory under `identifier`. -Result> CreateManagedTable(const std::shared_ptr& file_system, - const std::string& path, - const Identifier& identifier, - const std::string& branch = "main") { +// `branch`'s directory under `identifier` with `options`. +Result> CreateManagedTable( + const std::shared_ptr& file_system, const std::string& path, + const Identifier& identifier, const std::string& branch = "main", + const std::map& options = {}) { SchemaManager schema_manager(file_system, path, branch); auto schema = arrow::schema({arrow::field("id", arrow::int32(), /*nullable=*/false)}); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr created, schema_manager.CreateTable(schema, /*partition_keys=*/{}, - /*primary_keys=*/{}, /*options=*/{})); + /*primary_keys=*/{}, options)); return std::make_shared( file_system, path, - checked_pointer_cast(std::shared_ptr(std::move(created))), identifier); + checked_pointer_cast(std::shared_ptr(std::move(created))), identifier, + /*uuid=*/"", branch); } } // namespace +TEST(TableTest, TestTableCarriesManagersOfItsBranch) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + auto fs = dir->GetFileSystem(); + const std::string path = dir->Str() + "/tbl"; + ASSERT_OK_AND_ASSIGN( + std::shared_ptr
table, + CreateManagedTable(fs, path, Identifier(Identifier::kUnknownDatabase, "tbl"), + /*branch=*/"dev")); + auto file_store_table = std::dynamic_pointer_cast(table); + ASSERT_TRUE(file_store_table != nullptr); + // The branch the table was built for is the one its managers point at. + EXPECT_EQ(file_store_table->CurrentBranch(), "dev"); + ASSERT_TRUE(file_store_table->GetSchemaManager() != nullptr); + ASSERT_TRUE(file_store_table->GetSnapshotManager() != nullptr); + EXPECT_EQ(file_store_table->GetSnapshotManager()->Branch(), "dev"); + EXPECT_EQ(file_store_table->GetSnapshotManager()->RootPath(), path); + EXPECT_EQ(file_store_table->GetSnapshotManager()->Fs(), fs); + // A copy serves the same branch through the same managers, whose cache it shares. + ASSERT_OK_AND_ASSIGN(std::shared_ptr
copy, table->Copy({})); + auto file_store_copy = std::dynamic_pointer_cast(copy); + ASSERT_TRUE(file_store_copy != nullptr); + EXPECT_EQ(file_store_copy->CurrentBranch(), "dev"); + EXPECT_EQ(file_store_copy->GetSchemaManager(), file_store_table->GetSchemaManager()); + EXPECT_EQ(file_store_copy->GetSnapshotManager(), file_store_table->GetSnapshotManager()); +} + +TEST(TableTest, TestSwitchToBranch) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + auto fs = dir->GetFileSystem(); + const std::string path = dir->Str() + "/tbl"; + const Identifier identifier(Identifier::kUnknownDatabase, "tbl"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, CreateManagedTable(fs, path, identifier)); + // A branch of its own, with a schema of its own. + SchemaManager dev_schema_manager(fs, path, "dev"); + ASSERT_OK_AND_ASSIGN( + [[maybe_unused]] std::unique_ptr dev_schema, + dev_schema_manager.CreateTable(arrow::schema({arrow::field("c0", arrow::int32())}), + /*partition_keys=*/{}, /*primary_keys=*/{}, {})); + + // Switching to a branch serves that branch's schema and managers. + auto main_file_store_table = std::dynamic_pointer_cast(table); + ASSERT_TRUE(main_file_store_table != nullptr); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
dev_table, + main_file_store_table->SwitchToBranch("dev")); + auto dev_file_store_table = std::dynamic_pointer_cast(dev_table); + ASSERT_TRUE(dev_file_store_table != nullptr); + EXPECT_EQ("dev", dev_file_store_table->CurrentBranch()); + ASSERT_EQ((std::vector{"c0"}), dev_table->LatestSchema()->FieldNames()); + auto dev_table_schema = std::dynamic_pointer_cast(dev_table->LatestSchema()); + ASSERT_TRUE(dev_table_schema != nullptr); + ASSERT_EQ("dev", dev_table_schema->Options().at(Options::BRANCH)); + ASSERT_TRUE(dev_file_store_table->GetSchemaManager() != nullptr); + ASSERT_TRUE(dev_file_store_table->GetSnapshotManager() != nullptr); + EXPECT_EQ("dev", dev_file_store_table->GetSnapshotManager()->Branch()); + + // Switching to the branch already served hands back the same table, managers shared. + ASSERT_OK_AND_ASSIGN(std::shared_ptr
same_table, + dev_file_store_table->SwitchToBranch("dev")); + auto same_file_store_table = std::dynamic_pointer_cast(same_table); + ASSERT_TRUE(same_file_store_table != nullptr); + EXPECT_EQ(same_table->LatestSchema(), dev_table->LatestSchema()); + EXPECT_EQ(same_file_store_table->GetSchemaManager(), dev_file_store_table->GetSchemaManager()); + EXPECT_EQ(same_file_store_table->GetSnapshotManager(), + dev_file_store_table->GetSnapshotManager()); + + // Switching back serves the main branch's schema again. + ASSERT_OK_AND_ASSIGN(std::shared_ptr
main_again, + dev_file_store_table->SwitchToBranch("main")); + auto main_again_file_store_table = std::dynamic_pointer_cast(main_again); + ASSERT_TRUE(main_again_file_store_table != nullptr); + EXPECT_EQ("main", main_again_file_store_table->CurrentBranch()); + ASSERT_EQ((std::vector{"id"}), main_again->LatestSchema()->FieldNames()); + auto main_again_schema = std::dynamic_pointer_cast(main_again->LatestSchema()); + ASSERT_TRUE(main_again_schema != nullptr); + EXPECT_EQ(0, main_again_schema->Options().count(Options::BRANCH)); + + // A branch with no schema does not exist. + ASSERT_NOK_WITH_MSG(main_file_store_table->SwitchToBranch("missing"), "branch"); +} + +TEST(TableTest, TestNewWriteServesTheBranchTheTableWasLoadedWith) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + const std::shared_ptr fs = dir->GetFileSystem(); + const Identifier identifier("db", "tbl"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, + CreateManagedTable(fs, dir->Str(), identifier, /*branch=*/"dev", + {{Options::BUCKET, "1"}, {Options::BUCKET_KEY, "id"}})); + + // A context that names no branch is the one a caller hands straight to the table's write, + // which serves the branch the table was loaded with rather than the context's own main. + WriteContextBuilder write_builder(dir->Str(), "commit-user"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr write, + table->NewWrite(std::move(write_context))); + arrow::Int32Builder ids_builder; + ASSERT_TRUE(ids_builder.AppendValues({1, 2}).ok()); + std::shared_ptr ids_array; + ASSERT_TRUE(ids_builder.Finish(&ids_array).ok()); + auto struct_array = arrow::StructArray::Make({ids_array}, {arrow::field("id", arrow::int32(), + /*nullable=*/false)}) + .ValueOrDie(); + ::ArrowArray arrow_array; + ASSERT_TRUE(arrow::ExportArray(*struct_array, &arrow_array).ok()); + RecordBatchBuilder batch_builder(&arrow_array); + ASSERT_OK_AND_ASSIGN(std::unique_ptr batch, batch_builder.SetBucket(0).Finish()); + ASSERT_OK(write->Write(std::move(batch))); + ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::vector> messages, + write->PrepareCommit()); + ASSERT_OK(write->Close()); + + // The files land under the branch's own directory, which a write of the main branch - the + // branch the context alone would name - would never touch. + std::vector branch_entries; + ASSERT_OK(fs->ListDir(BranchManager::BranchPath(dir->Str(), "dev"), &branch_entries)); + EXPECT_FALSE(branch_entries.empty()); +} + +TEST(TableTest, TestOperatorsServeTheBranchTheTableWasLoadedWith) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + const Identifier identifier("db", "tbl"); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr
table, + CreateManagedTable(dir->GetFileSystem(), dir->Str(), identifier, /*branch=*/"dev", + {{Options::BUCKET, "1"}, {Options::BUCKET_KEY, "id"}})); + + // The write and the commit below carry no branch, and both serve the branch the table was + // loaded with, so the data they publish lands on it. + WriteContextBuilder write_builder(dir->Str(), "commit-user"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr write, + table->NewWrite(std::move(write_context))); + arrow::Int32Builder ids_builder; + ASSERT_TRUE(ids_builder.AppendValues({1, 2}).ok()); + std::shared_ptr ids_array; + ASSERT_TRUE(ids_builder.Finish(&ids_array).ok()); + auto struct_array = arrow::StructArray::Make({ids_array}, {arrow::field("id", arrow::int32(), + /*nullable=*/false)}) + .ValueOrDie(); + ::ArrowArray arrow_array; + ASSERT_TRUE(arrow::ExportArray(*struct_array, &arrow_array).ok()); + RecordBatchBuilder batch_builder(&arrow_array); + ASSERT_OK_AND_ASSIGN(std::unique_ptr batch, batch_builder.SetBucket(0).Finish()); + ASSERT_OK(write->Write(std::move(batch))); + ASSERT_OK_AND_ASSIGN(std::vector> messages, + write->PrepareCommit()); + ASSERT_OK(write->Close()); + CommitContextBuilder commit_builder(dir->Str(), "commit-user"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, + table->NewCommit(std::move(commit_context))); + ASSERT_OK(commit->Commit(messages)); + + // The scan finds what they wrote, reading through the table's own managers on its branch. + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_ctx, + ScanContextBuilder(dir->Str()).Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, table->NewScan(std::move(scan_ctx))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); + EXPECT_FALSE(plan->Splits().empty()); + + // A branch the scan context carries does not win either: the table's own does, so the data + // it finds is still the branch's. + ScanContextBuilder main_scan_builder(dir->Str()); + main_scan_builder.AddOption(Options::BRANCH, "main"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr main_scan_ctx, main_scan_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr main_scan, + table->NewScan(std::move(main_scan_ctx))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr main_plan, main_scan->CreatePlan()); + EXPECT_FALSE(main_plan->Splits().empty()); +} + TEST(TableTest, TestGetTableWithUnknownDatabase) { auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); @@ -166,9 +348,9 @@ TEST(TableTest, TestMainBranchAliasResolvesToTheMainBranch) { SCOPED_TRACE(main_identifier.ToString()); // A catalog that loaded the schema itself calls the constructor directly, which a // rest catalog is shaped like. The alias resolves to the main branch as well. - FileStoreTable table(fs, dir->Str(), schema, main_identifier); - ASSERT_OK_AND_ASSIGN(std::string branch, table.CurrentBranch()); - EXPECT_EQ("main", branch); + ASSERT_OK_AND_ASSIGN(std::string branch, main_identifier.GetBranchNameOrDefault()); + FileStoreTable table(fs, dir->Str(), schema, main_identifier, /*uuid=*/"", branch); + EXPECT_EQ("main", table.CurrentBranch()); } } @@ -180,15 +362,17 @@ TEST(TableTest, TestCurrentBranch) { ASSERT_OK_AND_ASSIGN( std::shared_ptr
main_table, CreateManagedTable(dir->GetFileSystem(), dir->Str(), Identifier("db", "tbl"))); - ASSERT_OK_AND_ASSIGN(std::string branch, main_table->CurrentBranch()); - EXPECT_EQ(branch, "main"); + auto main_file_store_table = std::dynamic_pointer_cast(main_table); + ASSERT_TRUE(main_file_store_table != nullptr); + EXPECT_EQ(main_file_store_table->CurrentBranch(), "main"); - // The identifier a table was opened with names the branch it is for. + // The branch a table was created on is the one it carries. ASSERT_OK_AND_ASSIGN(std::shared_ptr
branch_table, CreateManagedTable(dir->GetFileSystem(), dir->Str() + "/branch_tbl", Identifier("db", "tbl", "dev"), /*branch=*/"dev")); - ASSERT_OK_AND_ASSIGN(branch, branch_table->CurrentBranch()); - EXPECT_EQ(branch, "dev"); + auto branch_file_store_table = std::dynamic_pointer_cast(branch_table); + ASSERT_TRUE(branch_file_store_table != nullptr); + EXPECT_EQ(branch_file_store_table->CurrentBranch(), "dev"); } TEST(TableTest, TestCatalogOpenedTableServesThePerTableFileSystem) { @@ -314,6 +498,110 @@ TEST(TableTest, TestNewScanServesTheSchemaTheTableWasLoadedWith) { ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(ctx)), "not found latest schema"); } +TEST(TableTest, TestNewScanServesTheFileSystemTheTableWasOpenedWith) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + const std::shared_ptr fs = dir->GetFileSystem(); + const Identifier identifier("db", "tbl"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, + CreateManagedTable(fs, dir->Str(), identifier, /*branch=*/"main", + {{Options::BUCKET, "1"}, {Options::BUCKET_KEY, "id"}})); + + // A file system is only observable once a scan has data to read, so two rows are written and + // committed through the table's own write and commit. + WriteContextBuilder write_builder(dir->Str(), "commit-user"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr write, + table->NewWrite(std::move(write_context))); + arrow::Int32Builder ids_builder; + ASSERT_TRUE(ids_builder.AppendValues({1, 2}).ok()); + std::shared_ptr ids_array; + ASSERT_TRUE(ids_builder.Finish(&ids_array).ok()); + auto struct_array = arrow::StructArray::Make({ids_array}, {arrow::field("id", arrow::int32(), + /*nullable=*/false)}) + .ValueOrDie(); + ::ArrowArray arrow_array; + ASSERT_TRUE(arrow::ExportArray(*struct_array, &arrow_array).ok()); + RecordBatchBuilder batch_builder(&arrow_array); + ASSERT_OK_AND_ASSIGN(std::unique_ptr batch, batch_builder.SetBucket(0).Finish()); + ASSERT_OK(write->Write(std::move(batch))); + ASSERT_OK_AND_ASSIGN(std::vector> messages, + write->PrepareCommit()); + CommitContextBuilder commit_builder(dir->Str(), "commit-user"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, + table->NewCommit(std::move(commit_context))); + ASSERT_OK(commit->Commit(messages)); + ASSERT_OK(write->Close()); + + // The table was opened with the local file system, so a scan serves its data through that + // one even when the context hands over a mock that can read nothing. + ASSERT_OK_AND_ASSIGN( + std::unique_ptr held_ctx, + ScanContextBuilder(dir->Str()).WithFileSystem(std::make_shared()).Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, table->NewScan(std::move(held_ctx))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); + ASSERT_FALSE(plan->Splits().empty()); + + // A context without a table resolves the file system itself, which is the mock and reads no + // schema at all: the file system counterpart of the schema case above. + ASSERT_OK_AND_ASSIGN( + std::unique_ptr ctx, + ScanContextBuilder(dir->Str()).WithFileSystem(std::make_shared()).Finish()); + ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(ctx)), "not found latest schema"); +} + +TEST(TableTest, TestNewScanServesTheRootPathTheTableWasOpenedWith) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + const std::shared_ptr fs = dir->GetFileSystem(); + const Identifier identifier("db", "tbl"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, + CreateManagedTable(fs, dir->Str(), identifier, /*branch=*/"main", + {{Options::BUCKET, "1"}, {Options::BUCKET_KEY, "id"}})); + + // Two rows are written and committed through the table's own write and commit, so the scan + // below has data to find. + WriteContextBuilder write_builder(dir->Str(), "commit-user"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr write, + table->NewWrite(std::move(write_context))); + arrow::Int32Builder ids_builder; + ASSERT_TRUE(ids_builder.AppendValues({1, 2}).ok()); + std::shared_ptr ids_array; + ASSERT_TRUE(ids_builder.Finish(&ids_array).ok()); + auto struct_array = arrow::StructArray::Make({ids_array}, {arrow::field("id", arrow::int32(), + /*nullable=*/false)}) + .ValueOrDie(); + ::ArrowArray arrow_array; + ASSERT_TRUE(arrow::ExportArray(*struct_array, &arrow_array).ok()); + RecordBatchBuilder batch_builder(&arrow_array); + ASSERT_OK_AND_ASSIGN(std::unique_ptr batch, batch_builder.SetBucket(0).Finish()); + ASSERT_OK(write->Write(std::move(batch))); + ASSERT_OK_AND_ASSIGN(std::vector> messages, + write->PrepareCommit()); + CommitContextBuilder commit_builder(dir->Str(), "commit-user"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, + table->NewCommit(std::move(commit_context))); + ASSERT_OK(commit->Commit(messages)); + ASSERT_OK(write->Close()); + + // The table was opened with this directory as its root path, so a scan serves its data from + // there even when the context names a different one. + ASSERT_OK_AND_ASSIGN(std::unique_ptr held_ctx, + ScanContextBuilder(dir->Str() + "/not-the-table").Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, table->NewScan(std::move(held_ctx))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); + ASSERT_FALSE(plan->Splits().empty()); + + // A context without a table reads the schema from the directory it names, which is not a + // table at all: the root path counterpart of the schema case above. + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, + ScanContextBuilder(dir->Str() + "/not-the-table").Finish()); + ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(ctx)), "not found latest schema"); +} + TEST(TableTest, TestNewReadCreatesATableRead) { auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); @@ -383,6 +671,122 @@ TEST(TableTest, TestNewScanReadWriteCommitRejectANullContext) { ASSERT_NOK_WITH_MSG(table->NewCommit(nullptr), "commit context is null pointer"); } +TEST(TableTest, TestOperatorsRefuseAnOptionThatContradictsTheTable) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + const Identifier identifier("db", "tbl"); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr
table, + CreateManagedTable(dir->GetFileSystem(), dir->Str(), identifier, /*branch=*/"main", + {{Options::BUCKET, "1"}, {Options::BUCKET_KEY, "id"}})); + + // The table was loaded with `bucket` = 1, so every operator refuses a context that names + // another value for it: the table serves the options it was loaded with, and a caller that + // means to change them copies the table. + { + ScanContextBuilder scan_builder(dir->Str()); + scan_builder.AddOption(Options::BUCKET, "2"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, scan_builder.Finish()); + ASSERT_NOK_WITH_MSG(table->NewScan(std::move(ctx)), + "option 'bucket' is '2' in the context but '1' on the table"); + } + { + ReadContextBuilder read_builder(dir->Str()); + read_builder.AddOption(Options::BUCKET, "2"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, read_builder.Finish()); + ASSERT_NOK_WITH_MSG(table->NewRead(std::move(ctx)), + "option 'bucket' is '2' in the context but '1' on the table"); + } + { + WriteContextBuilder write_builder(dir->Str(), "commit-user"); + write_builder.AddOption(Options::BUCKET, "2"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, write_builder.Finish()); + ASSERT_NOK_WITH_MSG(table->NewWrite(std::move(ctx)), + "option 'bucket' is '2' in the context but '1' on the table"); + } + { + CommitContextBuilder commit_builder(dir->Str(), "commit-user"); + commit_builder.AddOption(Options::BUCKET, "2"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, commit_builder.Finish()); + ASSERT_NOK_WITH_MSG(table->NewCommit(std::move(ctx)), + "option 'bucket' is '2' in the context but '1' on the table"); + } + + // The copy made with the option in question agrees with the context, so it serves it. + ASSERT_OK_AND_ASSIGN(std::shared_ptr
copied, table->Copy({{Options::BUCKET, "2"}})); + ScanContextBuilder agreeing_scan_builder(dir->Str()); + agreeing_scan_builder.AddOption(Options::BUCKET, "2"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, agreeing_scan_builder.Finish()); + ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr scan, + copied->NewScan(std::move(ctx))); +} + +TEST(TableTest, TestCopyServesTheTableWithTheGivenOptionsOnTop) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + const std::shared_ptr fs = dir->GetFileSystem(); + const Identifier identifier("db", "tbl"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, + CreateManagedTable(fs, dir->Str(), identifier, /*branch=*/"main", + {{Options::BUCKET, "1"}, {Options::BUCKET_KEY, "id"}})); + + // Two rows are written and committed through the table's own write and commit, so the copy + // below has data to find. + WriteContextBuilder write_builder(dir->Str(), "commit-user"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr write, + table->NewWrite(std::move(write_context))); + arrow::Int32Builder ids_builder; + ASSERT_TRUE(ids_builder.AppendValues({1, 2}).ok()); + std::shared_ptr ids_array; + ASSERT_TRUE(ids_builder.Finish(&ids_array).ok()); + auto struct_array = arrow::StructArray::Make({ids_array}, {arrow::field("id", arrow::int32(), + /*nullable=*/false)}) + .ValueOrDie(); + ::ArrowArray arrow_array; + ASSERT_TRUE(arrow::ExportArray(*struct_array, &arrow_array).ok()); + RecordBatchBuilder batch_builder(&arrow_array); + ASSERT_OK_AND_ASSIGN(std::unique_ptr batch, batch_builder.SetBucket(0).Finish()); + ASSERT_OK(write->Write(std::move(batch))); + ASSERT_OK_AND_ASSIGN(std::vector> messages, + write->PrepareCommit()); + CommitContextBuilder commit_builder(dir->Str(), "commit-user"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, + table->NewCommit(std::move(commit_context))); + ASSERT_OK(commit->Commit(messages)); + ASSERT_OK(write->Close()); + + // The copy overlays what the call gives on the options the schema stored; the schema itself + // and the identity the table was opened with are copied as they are. + ASSERT_OK_AND_ASSIGN(std::shared_ptr
copied, + table->Copy({{Options::BUCKET, "2"}, {Options::TYPE, "format-table"}})); + std::shared_ptr copied_schema = + std::dynamic_pointer_cast(copied->LatestSchema()); + ASSERT_TRUE(copied_schema != nullptr); + EXPECT_EQ(copied_schema->Options().at(Options::BUCKET), "2"); + // `type` is structural and read from the schema alone, so the copy carries none of it: the + // option given at the call does not restyle a managed table. + EXPECT_EQ(copied_schema->Options().count(Options::TYPE), 0); + EXPECT_EQ(copied_schema->Options().at(Options::BUCKET_KEY), "id"); + EXPECT_EQ(copied_schema->Id(), + std::dynamic_pointer_cast(table->LatestSchema())->Id()); + EXPECT_EQ(copied->FullName(), table->FullName()); + EXPECT_EQ(copied->GetRootPath(), table->GetRootPath()); + EXPECT_EQ(copied->GetFileSystem(), table->GetFileSystem()); + // The original keeps the options it was loaded with. + EXPECT_EQ( + std::dynamic_pointer_cast(table->LatestSchema())->Options().at(Options::BUCKET), + "1"); + + // The copy is a table like the one it came from, and serves its data as one. + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_ctx, + ScanContextBuilder(dir->Str()).Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, copied->NewScan(std::move(scan_ctx))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); + ASSERT_FALSE(plan->Splits().empty()); +} + TEST(TableTest, TestCreateBuildsTheOperator) { auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); diff --git a/src/paimon/rest/rest_catalog.cpp b/src/paimon/rest/rest_catalog.cpp index 3ff2ed756..32ebefc1e 100644 --- a/src/paimon/rest/rest_catalog.cpp +++ b/src/paimon/rest/rest_catalog.cpp @@ -468,8 +468,12 @@ Result> RestCatalog::GetTable(const Identifier& identifie // tokens binds to this table. PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table_file_system, GetTableFileSystem(identifier)); - return std::make_shared(table_file_system, table_path, schema, identifier, - uuid); + // The branch the identifier carries, which the schema and path above were read on, so the + // table and its managers serve it. + PAIMON_ASSIGN_OR_RAISE(std::optional branch, identifier.GetBranchName()); + return std::make_shared( + table_file_system, table_path, schema, identifier, uuid, + NormalizeBranch(std::move(branch)).value_or(Identifier::kDefaultMainBranch)); } std::string RestCatalog::GetRootPath() const { diff --git a/src/paimon/rest/rest_catalog_test.cpp b/src/paimon/rest/rest_catalog_test.cpp index c76aa4b33..68081328e 100644 --- a/src/paimon/rest/rest_catalog_test.cpp +++ b/src/paimon/rest/rest_catalog_test.cpp @@ -44,6 +44,7 @@ #include "paimon/core/catalog/commit_table_request.h" #include "paimon/core/partition/partition_statistics.h" #include "paimon/core/schema/table_schema.h" +#include "paimon/core/table/file_store_table.h" #include "paimon/core/table/system/system_table.h" #include "paimon/core/utils/snapshot_manager.h" #include "paimon/defs.h" @@ -1072,8 +1073,9 @@ TEST_F(RestCatalogTest, BranchTableLoadsBranchSchemaFromServer) { ASSERT_EQ(0, main_case_schema->Options().count(Options::BRANCH)); // the branch the alias spells resolves to the main branch, not to a branch // directory named after the alias - ASSERT_OK_AND_ASSIGN(std::string main_case_branch, main_case_table->CurrentBranch()); - ASSERT_EQ("main", main_case_branch); + auto main_case_file_store_table = std::dynamic_pointer_cast(main_case_table); + ASSERT_NE(nullptr, main_case_file_store_table); + ASSERT_EQ("main", main_case_file_store_table->CurrentBranch()); // a system table on a branch resolves against the branch's data table: the system // suffix is stripped while the branch stays in the identifier sent to the server diff --git a/test/inte/realtime_write_inte_test.cpp b/test/inte/realtime_write_inte_test.cpp index 5de6c9fa7..fd9b49650 100644 --- a/test/inte/realtime_write_inte_test.cpp +++ b/test/inte/realtime_write_inte_test.cpp @@ -43,6 +43,7 @@ #include "gtest/gtest.h" #include "paimon/catalog/catalog.h" #include "paimon/catalog/identifier.h" +#include "paimon/catalog/table.h" #include "paimon/commit_context.h" #include "paimon/common/data/variant/variant_type_utils.h" #include "paimon/common/table/special_fields.h" @@ -559,6 +560,41 @@ class RealtimeWriteInteTest : public ::testing::Test { return FileStoreWrite::Create(std::move(context)); } + // A writer that means to change an option the table was created with writes through a copy + // carrying it, which the context then agrees with. + Result> CopiedTable( + const std::map& overrides) const { + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr catalog, Catalog::Create(dir_->Str(), {})); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, + catalog->GetTable(Identifier("foo", "bar"))); + return table->Copy(overrides); + } + + // Evolves the table schema to the current options with its fields unchanged, the way an + // ALTER TABLE changing an option would: the table itself then serves the changed option. + // The fields are carried over from the latest schema so their attributes survive: rebuilt + // raw fields would lose the non-nullable primary keys the table schema enforces. + Status EvolveSchemaWithOptions() const { + SchemaManager schema_manager(dir_->GetFileSystem(), table_path_); + PAIMON_ASSIGN_OR_RAISE(std::optional> latest, + schema_manager.Latest()); + if (!latest.has_value()) { + return Status::Invalid("table schema does not exist"); + } + return TestHelper::WriteNextSchema(dir_->GetFileSystem(), table_path_, + latest.value()->Fields(), + latest.value()->HighestFieldId(), options_); + } + + Result> CreateRealtimeWriter( + const std::shared_ptr& realtime_context, + const std::shared_ptr
& table) const { + WriteContextBuilder builder(table_path_, commit_user_); + builder.SetOptions(options_).WithStreamingMode(true).WithRealtimeContext(realtime_context); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, builder.Finish()); + return table->NewWrite(std::move(context)); + } + Result> CreateRealtimeWriter() const { PAIMON_ASSIGN_OR_RAISE(std::shared_ptr realtime_context, RealtimeContext::Create()); @@ -1249,32 +1285,36 @@ TEST_F(RealtimeWriteInteTest, TestRealtimeOperationsRequireEnabledOption) { disabled_options[Options::REALTIME_ENABLED] = "false"; ASSERT_OK_AND_ASSIGN(std::shared_ptr realtime_context, RealtimeContext::Create()); + // The table was created enabled, so each operation below goes through a copy carrying it + // disabled, whose contexts then agree: each reports that real-time needs the option itself. + ASSERT_OK_AND_ASSIGN(std::shared_ptr
disabled_table, + CopiedTable({{Options::REALTIME_ENABLED, "false"}})); WriteContextBuilder write_builder(table_path_, commit_user_); write_builder.SetOptions(disabled_options) .WithStreamingMode(true) .WithRealtimeContext(realtime_context); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); - ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(std::move(write_context)), + ASSERT_NOK_WITH_MSG(disabled_table->NewWrite(std::move(write_context)), "real-time write requires realtime.enabled=true"); ScanContextBuilder scan_builder(table_path_); scan_builder.SetOptions(disabled_options).WithRealtimeContext(realtime_context); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_builder.Finish()); - ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(scan_context)), + ASSERT_NOK_WITH_MSG(disabled_table->NewScan(std::move(scan_context)), "real-time scan requires realtime.enabled=true"); ReadContextBuilder read_builder(table_path_); read_builder.SetOptions(disabled_options).WithRealtimeContext(realtime_context); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); - ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(read_context)), + ASSERT_NOK_WITH_MSG(disabled_table->NewRead(std::move(read_context)), "real-time read requires realtime.enabled=true"); CommitContextBuilder commit_builder(table_path_, commit_user_); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.SetOptions(disabled_options).Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, - FileStoreCommit::Create(std::move(commit_context))); + disabled_table->NewCommit(std::move(commit_context))); ASSERT_NOK_WITH_MSG(commit->CommitWithProgress(/*realtime_commits=*/{}, /*commit_identifier=*/0, /*watermark=*/std::nullopt), @@ -2084,6 +2124,9 @@ TEST_F(RealtimeWriteInteTest, TestRealtimeDeduplicateAcrossMultipleFileIndexStor ASSERT_OK(embedded_writer->Close()); options_[Options::FILE_INDEX_IN_MANIFEST_THRESHOLD] = "1B"; + // The tighter threshold is a change the table itself carries, the way an ALTER TABLE would: + // the schema is evolved to it, so every later write, commit, and read serves it. + ASSERT_OK(EvolveSchemaWithOptions()); ASSERT_OK_AND_ASSIGN(std::unique_ptr external_writer, CreateRealtimeWriter(realtime_context)); ASSERT_OK_AND_ASSIGN(std::unique_ptr external_batch, @@ -3766,6 +3809,9 @@ TEST_F(RealtimeWriteInteTest, TestAppendRealtimeReadUsesEmbeddedAndExternalBitma ASSERT_OK(embedded_writer->Close()); options_[Options::FILE_INDEX_IN_MANIFEST_THRESHOLD] = "1B"; + // The tighter threshold is a change the table itself carries, the way an ALTER TABLE would: + // the schema is evolved to it, so every later write, commit, and read serves it. + ASSERT_OK(EvolveSchemaWithOptions()); ASSERT_OK_AND_ASSIGN(std::unique_ptr external_writer, CreateRealtimeWriter(realtime_context)); ASSERT_OK_AND_ASSIGN( @@ -4083,6 +4129,9 @@ TEST_F(RealtimeWriteInteTest, TestPkDvRealtimeReadUsesEmbeddedAndExternalBitmapI ASSERT_OK(embedded_writer->Close()); options_[Options::FILE_INDEX_IN_MANIFEST_THRESHOLD] = "1B"; + // The tighter threshold is a change the table itself carries, the way an ALTER TABLE would: + // the schema is evolved to it, so every later write, commit, and read serves it. + ASSERT_OK(EvolveSchemaWithOptions()); ASSERT_OK_AND_ASSIGN(std::unique_ptr external_writer, CreateRealtimeWriter(realtime_context)); ASSERT_OK_AND_ASSIGN(std::unique_ptr external_batch, diff --git a/test/inte/scan_and_read_inte_test.cpp b/test/inte/scan_and_read_inte_test.cpp index b7cc73b96..41d35de8d 100644 --- a/test/inte/scan_and_read_inte_test.cpp +++ b/test/inte/scan_and_read_inte_test.cpp @@ -36,6 +36,9 @@ #include "fmt/ranges.h" #include "gtest/gtest.h" #include "paimon/bucket/bucket_id_calculator.h" +#include "paimon/catalog/catalog.h" +#include "paimon/catalog/identifier.h" +#include "paimon/catalog/table.h" #include "paimon/common/factories/io_hook.h" #include "paimon/common/io/cache/lru_cache.h" #include "paimon/common/table/special_fields.h" @@ -796,19 +799,27 @@ TEST_P(ScanAndReadInteTest, TestWithAppendBucketKeyPointLookup) { } ASSERT_GE(other_index, 0); // These keys share a current two-bucket selection but need different historical buckets. + // The data was written with four buckets, so reading a rescaled layout reads through a + // copy of the table carrying the rescaled bucket count: an option on the context would + // contradict the table the catalog loaded. + ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, + Catalog::Create(test_dir->Str(), {})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, + catalog->GetTable(Identifier("foo", "bar"))); for (int32_t lookup_index : {32, other_index}) { - // The data was written with four buckets. Read it with rescaled table options too. for (const std::string current_bucket_count : {"2", "4", "8", "17"}) { SCOPED_TRACE(current_bucket_count); const std::string lookup_key = fmt::format("key{:03}", lookup_index); auto lookup_predicate = PredicateBuilder::Equal( 0, "rowkey", field_type, Literal(field_type, lookup_key.data(), lookup_key.size())); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
rescaled_table, + table->Copy({{Options::BUCKET, current_bucket_count}})); ScanContextBuilder scan_builder(table_path); scan_builder.SetPredicate(lookup_predicate) .AddOption(Options::BUCKET, current_bucket_count); ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_builder)); - ASSERT_OK_AND_ASSIGN(auto scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto scan, rescaled_table->NewScan(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(auto plan, scan->CreatePlan()); if (EnableSnapshotLiveManifestCache()) { ASSERT_OK_AND_ASSIGN( @@ -820,7 +831,7 @@ TEST_P(ScanAndReadInteTest, TestWithAppendBucketKeyPointLookup) { .AddOption(Options::BUCKET, current_bucket_count); ASSERT_OK_AND_ASSIGN(auto cached_context, FinishScanContext(cached_builder)); ASSERT_OK_AND_ASSIGN(auto cached_scan, - TableScan::Create(std::move(cached_context))); + rescaled_table->NewScan(std::move(cached_context))); ASSERT_OK_AND_ASSIGN(plan, cached_scan->CreatePlan()); ASSERT_OK_AND_ASSIGN(uint64_t hit, cached_scan->GetMetrics()->GetCounter( ScanMetrics::LAST_SNAPSHOT_CACHE_HIT)); @@ -3086,12 +3097,21 @@ TEST_P(ScanAndReadInteTest, TestScanAndReadWithDisableIndex) { PredicateBuilder::Equal(/*field_index=*/0, /*field_name=*/"f0", FieldType::STRING, Literal(FieldType::STRING, "Alice0", 6)); + // The table was loaded with its index enabled, so the index-disabled pass below goes through + // a copy carrying it disabled: an option on the context would contradict the table. + ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, + Catalog::Create(paimon::test::GetDataDir() + file_format, {})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, + catalog->GetTable(Identifier("append_with_bitmap", "append_with_bitmap"))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
disabled_table, + table->Copy({{"file-index.read.enabled", "false"}})); + // scan ScanContextBuilder scan_context_builder(table_path); ASSERT_OK_AND_ASSIGN( auto scan_context, scan_context_builder.AddOption("file-index.read.enabled", "false").Finish()); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto table_scan, disabled_table->NewScan(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 1); @@ -3102,7 +3122,7 @@ TEST_P(ScanAndReadInteTest, TestScanAndReadWithDisableIndex) { read_context_builder.AddOption("file-index.read.enabled", "false"); AddReadOptionsForPrefetch(&read_context_builder); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, disabled_table->NewRead(std::move(read_context))); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(splits)); ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -3158,6 +3178,13 @@ TEST(ScanAndReadFileIndexInteTest, TestPkDvEmbeddedBitmapFiltersScanAndRead) { helper->WriteAndCommit(std::move(batch), /*commit_identifier=*/0, std::nullopt)); ASSERT_FALSE(commit_messages.empty()); + // The statistics-only baseline below disables the file index the table carries, so it scans + // through a copy with it disabled: an option on the context would contradict the table. + ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, Catalog::Create(dir->Str(), {})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, catalog->GetTable(Identifier("foo", "bar"))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
stats_only_table, + table->Copy({{Options::FILE_INDEX_READ_ENABLED, "false"}})); + // 15 falls inside the file's [10, 30] min/max range, so statistics alone must retain the // file. Disabling file-index reads verifies that baseline explicitly. const std::shared_ptr absent_predicate = PredicateBuilder::Equal( @@ -3169,7 +3196,7 @@ TEST(ScanAndReadFileIndexInteTest, TestPkDvEmbeddedBitmapFiltersScanAndRead) { ASSERT_OK_AND_ASSIGN(std::unique_ptr stats_only_scan_context, stats_only_scan_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr stats_only_scan, - TableScan::Create(std::move(stats_only_scan_context))); + stats_only_table->NewScan(std::move(stats_only_scan_context))); ASSERT_OK_AND_ASSIGN(std::shared_ptr stats_only_plan, stats_only_scan->CreatePlan()); ASSERT_FALSE(stats_only_plan->Splits().empty()); @@ -3262,6 +3289,13 @@ TEST(ScanAndReadFileIndexInteTest, TestPkDvReconstructsFileIndexPredicateAfterSc const std::shared_ptr predicate = PredicateBuilder::Equal( /*field_index=*/1, /*field_name=*/"renamed_value", FieldType::INT, Literal(15)); + // The statistics-only baseline below disables the file index the table carries, so it scans + // through a copy with it disabled: an option on the context would contradict the table. + ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, Catalog::Create(dir->Str(), {})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, catalog->GetTable(Identifier("foo", "bar"))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
stats_only_table, + table->Copy({{Options::FILE_INDEX_READ_ENABLED, "false"}})); + // The evolved min/max range is still [10, 30], so statistics alone retain the old file. ScanContextBuilder stats_only_scan_builder(table_path); stats_only_scan_builder.SetOptions(options) @@ -3270,7 +3304,7 @@ TEST(ScanAndReadFileIndexInteTest, TestPkDvReconstructsFileIndexPredicateAfterSc ASSERT_OK_AND_ASSIGN(std::unique_ptr stats_only_scan_context, stats_only_scan_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr stats_only_scan, - TableScan::Create(std::move(stats_only_scan_context))); + stats_only_table->NewScan(std::move(stats_only_scan_context))); ASSERT_OK_AND_ASSIGN(std::shared_ptr stats_only_plan, stats_only_scan->CreatePlan()); ASSERT_FALSE(stats_only_plan->Splits().empty()); @@ -4021,16 +4055,25 @@ TEST_P(ScanAndReadInteTest, TestWithPKBucketSelectByPredicate) { auto predicate = PredicateBuilder::Equal(/*field_index=*/2, /*field_name=*/"f2", FieldType::INT, Literal(static_cast(0))); - // Historical files use two buckets even when the current option has changed. + // Historical files use two buckets even when the current option has changed, so the scan + // goes through a copy of the table carrying the bucket count it selects by: an option on + // the context would contradict the table the catalog loaded. + ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, + Catalog::Create(paimon::test::GetDataDir() + file_format, {})); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr
table, + catalog->GetTable(Identifier("pk_table_scan_and_read_dv", "pk_table_scan_and_read_dv"))); for (const std::string current_bucket_count : {"2", "4", "8", "17"}) { SCOPED_TRACE(current_bucket_count); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
rescaled_table, + table->Copy({{Options::BUCKET, current_bucket_count}})); ScanContextBuilder scan_context_builder(table_path); scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "6"); scan_context_builder.AddOption(Options::BUCKET, current_bucket_count); scan_context_builder.SetPartitionFilter({{{"f1", "10"}}}); scan_context_builder.SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto table_scan, rescaled_table->NewScan(std::move(scan_context))); ReadContextBuilder read_context_builder(table_path); AddReadOptionsForPrefetch(&read_context_builder); @@ -4049,7 +4092,8 @@ TEST_P(ScanAndReadInteTest, TestWithPKBucketSelectByPredicate) { .SetPartitionFilter({{{"f1", "10"}}}) .SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(auto cached_context, FinishScanContext(cached_builder)); - ASSERT_OK_AND_ASSIGN(auto cached_scan, TableScan::Create(std::move(cached_context))); + ASSERT_OK_AND_ASSIGN(auto cached_scan, + rescaled_table->NewScan(std::move(cached_context))); ASSERT_OK_AND_ASSIGN(result_plan, cached_scan->CreatePlan()); ASSERT_OK_AND_ASSIGN(uint64_t hit, cached_scan->GetMetrics()->GetCounter( ScanMetrics::LAST_SNAPSHOT_CACHE_HIT)); diff --git a/test/inte/scan_inte_test.cpp b/test/inte/scan_inte_test.cpp index 3f949484d..1502c39a7 100644 --- a/test/inte/scan_inte_test.cpp +++ b/test/inte/scan_inte_test.cpp @@ -32,6 +32,9 @@ #include "arrow/api.h" #include "gtest/gtest.h" +#include "paimon/catalog/catalog.h" +#include "paimon/catalog/identifier.h" +#include "paimon/catalog/table.h" #include "paimon/common/data/binary_row.h" #include "paimon/common/data/data_define.h" #include "paimon/common/utils/decimal_utils.h" @@ -1161,10 +1164,18 @@ TEST_P(ScanInteTest, TestScanAppendWithInvalidOptions) { "scan.snapshot-id or scan.tag-name must be set when startup mode is FROM_SNAPSHOT"); } { + // `bucket` = -2 is refused by the scan itself, which a copy of the table carrying it + // reaches: an option on the context would contradict the table the catalog loaded, + // which a scan through that table refuses. + ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, + Catalog::Create(paimon::test::GetDataDir() + "orc", {})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, + catalog->GetTable(Identifier("append_09", "append_09"))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
copied, table->Copy({{Options::BUCKET, "-2"}})); ScanContextBuilder context_builder(table_path); - context_builder.AddOption(Options::BUCKET, "-2").WithStreamingMode(true); + context_builder.WithStreamingMode(true); ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(scan_context)), + ASSERT_NOK_WITH_MSG(copied->NewScan(std::move(scan_context)), "do not support bucket=-2 in scan process"); } } @@ -1726,10 +1737,18 @@ TEST_P(ScanInteTest, TestScanPKWithSnapshot1WithBucketStats) { TEST_P(ScanInteTest, TestScanPKWithInvalidOptions) { std::string table_path = paimon::test::GetDataDir() + "orc/pk_table_with_total_buckets.db/pk_table_with_total_buckets"; + // Like the append case above, the invalid bucket count rides on a copy of the table: the + // scan refuses it where the bucket count is validated. + ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, + Catalog::Create(paimon::test::GetDataDir() + "orc", {})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, + catalog->GetTable(Identifier("pk_table_with_total_buckets", + "pk_table_with_total_buckets"))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
copied, table->Copy({{Options::BUCKET, "-1"}})); ScanContextBuilder context_builder(table_path); - context_builder.AddOption(Options::BUCKET, "-1").WithStreamingMode(true); + context_builder.WithStreamingMode(true); ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(scan_context)), + ASSERT_NOK_WITH_MSG(copied->NewScan(std::move(scan_context)), "do not support pk table bucket=-1 in scan process"); } diff --git a/test/inte/write_and_read_inte_with_index_test.cpp b/test/inte/write_and_read_inte_with_index_test.cpp index efda3de03..9123a3385 100644 --- a/test/inte/write_and_read_inte_with_index_test.cpp +++ b/test/inte/write_and_read_inte_with_index_test.cpp @@ -1250,8 +1250,7 @@ TEST_P(WriteAndReadInteWithIndexTest, TestReadWithLimits) { PredicateBuilder::NotEqual(/*field_index=*/0, /*field_name=*/"f0", FieldType::STRING, Literal(FieldType::STRING, "Alice", 5)); ReadContextBuilder context_builder(path); - context_builder.AddOption(Options::FILE_FORMAT, "orc") - .AddOption("test.enable-adaptive-prefetch-strategy", "false") + context_builder.AddOption("test.enable-adaptive-prefetch-strategy", "false") .AddOption(Options::READ_BATCH_SIZE, "1") .AddOption("orc.read.enable-metrics", "true") .SetPredicate(predicate); From 54367bb7bcc18fc1fa9f4b66b0afae9435b82dd7 Mon Sep 17 00:00:00 2001 From: "yonghao.fyh" Date: Thu, 22 Oct 2026 17:16:15 +0800 Subject: [PATCH 05/17] tmp --- src/paimon/CMakeLists.txt | 4 +- .../core/catalog/file_system_catalog.cpp | 5 +- .../table/append_only_file_store_table.cpp | 209 ++++++ .../core/table/append_only_file_store_table.h | 91 +++ src/paimon/core/table/file_store.cpp | 75 ++- src/paimon/core/table/file_store.h | 56 +- .../table/{table.cpp => file_store_table.cpp} | 596 ++++-------------- src/paimon/core/table/file_store_table.h | 124 +++- .../table/primary_key_file_store_table.cpp | 291 +++++++++ .../core/table/primary_key_file_store_table.h | 91 +++ src/paimon/core/table/table_test.cpp | 10 +- src/paimon/rest/rest_catalog.cpp | 2 +- src/paimon/testing/mock/mock_catalog.h | 2 +- 13 files changed, 1018 insertions(+), 538 deletions(-) create mode 100644 src/paimon/core/table/append_only_file_store_table.cpp create mode 100644 src/paimon/core/table/append_only_file_store_table.h rename src/paimon/core/table/{table.cpp => file_store_table.cpp} (66%) create mode 100644 src/paimon/core/table/primary_key_file_store_table.cpp create mode 100644 src/paimon/core/table/primary_key_file_store_table.h diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt index 261987aea..8d1234c4d 100644 --- a/src/paimon/CMakeLists.txt +++ b/src/paimon/CMakeLists.txt @@ -443,9 +443,10 @@ set(PAIMON_CORE_SRCS core/stats/simple_stats_converter.cpp core/stats/simple_stats.cpp core/stats/simple_stats_evolution.cpp - core/table/table.cpp + core/table/append_only_file_store_table.cpp core/table/bucket_mode.cpp core/table/file_store.cpp + core/table/file_store_table.cpp core/table/format/format_file_listing.cpp core/table/format/format_file_naming.cpp core/table/format/format_path_validation.cpp @@ -457,6 +458,7 @@ set(PAIMON_CORE_SRCS core/table/format/format_table_scan.cpp core/table/format/format_table_write.cpp core/table/format/lazy_concat_batch_reader.cpp + core/table/primary_key_file_store_table.cpp core/table/sink/commit_message.cpp core/table/sink/commit_message_impl.cpp core/table/sink/commit_message_serializer.cpp diff --git a/src/paimon/core/catalog/file_system_catalog.cpp b/src/paimon/core/catalog/file_system_catalog.cpp index 36721a148..d3c6bd92e 100644 --- a/src/paimon/core/catalog/file_system_catalog.cpp +++ b/src/paimon/core/catalog/file_system_catalog.cpp @@ -350,9 +350,8 @@ Result> FileSystemCatalog::GetTable(const Identifier& ide // The branch the schema above was read on, which the table carries so its managers read // the same branch: `GetBranchNameOrDefault` folds any spelling of `main`. PAIMON_ASSIGN_OR_RAISE(std::string branch, identifier.GetBranchNameOrDefault()); - return std::make_shared(fs_, table_path, - checked_pointer_cast(*latest_schema), - identifier, /*uuid=*/"", branch); + return FileStoreTable::Create(fs_, table_path, checked_pointer_cast(*latest_schema), + identifier, /*uuid=*/"", branch); } Result> FileSystemCatalog::LoadSystemTable( diff --git a/src/paimon/core/table/append_only_file_store_table.cpp b/src/paimon/core/table/append_only_file_store_table.cpp new file mode 100644 index 000000000..0846aef90 --- /dev/null +++ b/src/paimon/core/table/append_only_file_store_table.cpp @@ -0,0 +1,209 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/table/append_only_file_store_table.h" + +#include +#include +#include +#include +#include + +#include "fmt/format.h" +#include "paimon/core/deletionvectors/bucketed_dv_maintainer.h" +#include "paimon/core/index/index_file_handler.h" +#include "paimon/core/operation/append_only_file_store_scan.h" +#include "paimon/core/operation/append_only_file_store_write.h" +#include "paimon/core/operation/commit/commit_scanner.h" +#include "paimon/core/operation/data_evolution_file_store_scan.h" +#include "paimon/core/operation/file_store_scan.h" +#include "paimon/core/realtime/realtime_schema_layout.h" +#include "paimon/core/snapshot.h" +#include "paimon/core/table/bucket_mode.h" +#include "paimon/core/table/file_store.h" +#include "paimon/core/table/source/append_only_split_generator.h" +#include "paimon/core/table/source/append_only_table_read.h" +#include "paimon/core/table/source/data_evolution_split_generator.h" +#include "paimon/core/table/source/split_generator.h" +#include "paimon/core/utils/snapshot_manager.h" +#include "paimon/result.h" +#include "paimon/status.h" +#include "paimon/write_context.h" + +namespace paimon { + +AppendOnlyFileStoreTable::AppendOnlyFileStoreTable( + const std::shared_ptr& file_system, const std::string& root_path, + const std::shared_ptr& schema, const Identifier& identifier, const std::string& uuid, + const std::string& branch, const std::shared_ptr& schema_manager, + const std::shared_ptr& snapshot_manager) + : FileStoreTable(file_system, root_path, schema, identifier, uuid, branch, schema_manager, + snapshot_manager) {} + +Result> AppendOnlyFileStoreTable::CreateWrite( + const WriteContext& ctx, const std::shared_ptr& schema, + const std::shared_ptr& arrow_schema, + const std::shared_ptr& partition_schema, const CoreOptions& options, + const std::shared_ptr& path_factory, + const std::shared_ptr& snapshot_manager, + const std::shared_ptr& io_manager, bool ignore_previous_files) const { + // append table + const bool realtime_deduplicate = !options.GetRealtimeDeduplicateKeyFields().empty(); + bool need_dv_maintainer_factory = options.DeletionVectorsEnabled(); + if (options.GetBucket() == -1) { + need_dv_maintainer_factory = false; + ignore_previous_files = true; + } else if (options.GetBucket() <= 0) { + return Status::Invalid( + fmt::format("not support bucket {} in append table", options.GetBucket())); + } + if (ctx.GetRealtimeContext()) { + if (!ctx.GetWriteSchema().empty()) { + return Status::Invalid("real-time append write does not support a custom schema"); + } + if (!realtime_deduplicate) { + PAIMON_RETURN_NOT_OK(RestoreRealtimeCommittedProgress(ctx.GetRealtimeContext(), + snapshot_manager, options)); + } + } else if (realtime_deduplicate) { + return Status::Invalid("real-time deduplicate write requires a real-time context"); + } + std::shared_ptr write_schema = arrow_schema; + const auto& write_field_names = ctx.GetWriteSchema(); + if (!write_field_names.empty()) { + arrow::FieldVector write_fields; + write_fields.reserve(write_field_names.size()); + for (const auto& field_name : write_field_names) { + auto field = arrow_schema->GetFieldByName(field_name); + if (!field) { + // TODO(xinyu.lxy): support _ROW_ID and _SEQUENCE_NUMBER + return Status::Invalid( + fmt::format("write field {} does not exist in table schema", field_name)); + } + write_fields.push_back(field); + } + write_schema = arrow::schema(write_fields); + } + + std::shared_ptr realtime_schema_layout; + if (ctx.GetRealtimeContext()) { + const RealtimeStoreMode mode = + realtime_deduplicate ? RealtimeStoreMode::DEDUPLICATE : RealtimeStoreMode::APPEND_ONLY; + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr layout, + RealtimeSchemaLayout::Create(mode, write_schema)); + realtime_schema_layout = std::move(layout); + } + + std::shared_ptr dv_maintainer_factory; + if (need_dv_maintainer_factory) { + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr index_file_handler, + store_->NewIndexFileHandler(options, path_factory, ctx.GetMemoryPool())); + dv_maintainer_factory = + std::make_shared(std::move(index_file_handler)); + } + + auto file_store_write = std::make_unique( + path_factory, snapshot_manager, GetSchemaManager(), ctx.GetCommitUser(), root_path_, schema, + arrow_schema, write_schema, realtime_schema_layout, partition_schema, dv_maintainer_factory, + io_manager, options, ignore_previous_files, ctx.IsStreamingMode(), + ctx.IgnoreNumBucketCheck(), ctx.GetRealtimeContext(), ctx.GetExecutor(), + ctx.GetMemoryPool()); + if (ctx.GetRealtimeContext() && realtime_deduplicate) { + PAIMON_ASSIGN_OR_RAISE(std::optional latest_snapshot, + snapshot_manager->LatestSnapshot()); + if (latest_snapshot) { + PAIMON_RETURN_NOT_OK(file_store_write->RefreshCommittedSnapshot(latest_snapshot->Id())); + } + } + return std::unique_ptr(std::move(file_store_write)); +} + +Result> AppendOnlyFileStoreTable::NewFileStoreScan( + const std::shared_ptr& manifest_list, + const std::shared_ptr& manifest_file, + const std::shared_ptr& arrow_schema, + const std::shared_ptr& table_schema, const CoreOptions& core_options, + const std::shared_ptr& executor, const std::shared_ptr& memory_pool, + const ScanContext* context) const { + // TODO(liancheng.lsz): support fallback branch in scan + std::unique_ptr scan; + if (core_options.DataEvolutionEnabled()) { + PAIMON_ASSIGN_OR_RAISE( + scan, DataEvolutionFileStoreScan::Create(GetSnapshotManager(), GetSchemaManager(), + manifest_list, manifest_file, table_schema, + arrow_schema, context->GetScanFilters(), + core_options, executor, memory_pool)); + } else { + PAIMON_ASSIGN_OR_RAISE( + scan, AppendOnlyFileStoreScan::Create(GetSnapshotManager(), GetSchemaManager(), + manifest_list, manifest_file, table_schema, + arrow_schema, context->GetScanFilters(), + core_options, executor, memory_pool)); + } + return WithTablePath(std::move(scan), root_path_); +} + +Result> AppendOnlyFileStoreTable::NewSplitGenerator( + const std::shared_ptr& table_schema, const CoreOptions& core_options) const { + auto source_split_target_size = core_options.GetSourceSplitTargetSize(); + auto source_split_open_file_cost = core_options.GetSourceSplitOpenFileCost(); + if (core_options.DataEvolutionEnabled()) { + return std::make_unique(source_split_target_size, + source_split_open_file_cost, + core_options.BlobSplitByFileSize()); + } + BucketMode bucket_mode = + (core_options.GetBucket() == -1 ? BucketMode::BUCKET_UNAWARE : BucketMode::HASH_FIXED); + return std::make_unique(source_split_target_size, + source_split_open_file_cost, bucket_mode); +} + +Result> AppendOnlyFileStoreTable::NewTableRead( + const std::shared_ptr& path_factory, + const std::shared_ptr& internal_context, + const std::shared_ptr& memory_pool, + const std::shared_ptr& executor) const { + return std::make_unique(path_factory, internal_context, memory_pool, + executor); +} + +CommitScanner::ScanSupplier AppendOnlyFileStoreTable::NewCommitScanSupplier( + const std::shared_ptr& snapshot_manager, + const std::shared_ptr& manifest_list, + const std::shared_ptr& manifest_file, + const std::shared_ptr& table_schema, + const std::shared_ptr& arrow_schema, const CoreOptions& options, + const std::shared_ptr& executor, const std::shared_ptr& pool) const { + // The schema manager the table carries, which the scans this supplier hands out read the + // schema through. + const std::shared_ptr& schema_manager = GetSchemaManager(); + return [snapshot_manager, schema_manager, manifest_list, manifest_file, table_schema, + arrow_schema, options, executor, pool](const std::shared_ptr& scan_filter) + -> Result> { + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr scan, + AppendOnlyFileStoreScan::Create(snapshot_manager, schema_manager, manifest_list, + manifest_file, table_schema, arrow_schema, scan_filter, + options, executor, pool)); + return std::unique_ptr(std::move(scan)); + }; +} + +} // namespace paimon diff --git a/src/paimon/core/table/append_only_file_store_table.h b/src/paimon/core/table/append_only_file_store_table.h new file mode 100644 index 000000000..ea5374ba3 --- /dev/null +++ b/src/paimon/core/table/append_only_file_store_table.h @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include + +#include "paimon/catalog/identifier.h" +#include "paimon/core/table/file_store_table.h" +#include "paimon/core/utils/branch_manager.h" +#include "paimon/result.h" +#include "paimon/type_fwd.h" + +namespace paimon { + +class CoreOptions; +class FileStoreScan; +class IOManager; +class SplitGenerator; +class TableSchema; + +/// The managed table a schema without primary keys describes, Java Paimon's +/// `AppendOnlyFileStoreTable`. Its write appends the rows it is given, and its scan, read and +/// commit walk the files as they were written. The operator flows are the ones `FileStoreTable` +/// carries; the pieces those flows hand down are built here. +class AppendOnlyFileStoreTable : public FileStoreTable { + public: + /// The constructor `FileStoreTable::Create` builds an append-only table through; the + /// parameters are the ones it takes, and the base class carries them. + AppendOnlyFileStoreTable(const std::shared_ptr& file_system, + const std::string& root_path, const std::shared_ptr& schema, + const Identifier& identifier, const std::string& uuid = "", + const std::string& branch = BranchManager::DEFAULT_MAIN_BRANCH, + const std::shared_ptr& schema_manager = nullptr, + const std::shared_ptr& snapshot_manager = nullptr); + + private: + Result> CreateWrite( + const WriteContext& ctx, const std::shared_ptr& schema, + const std::shared_ptr& arrow_schema, + const std::shared_ptr& partition_schema, const CoreOptions& options, + const std::shared_ptr& path_factory, + const std::shared_ptr& snapshot_manager, + const std::shared_ptr& io_manager, bool ignore_previous_files) const override; + + Result> NewFileStoreScan( + const std::shared_ptr& manifest_list, + const std::shared_ptr& manifest_file, + const std::shared_ptr& arrow_schema, + const std::shared_ptr& table_schema, const CoreOptions& core_options, + const std::shared_ptr& executor, const std::shared_ptr& memory_pool, + const ScanContext* context) const override; + + Result> NewSplitGenerator( + const std::shared_ptr& table_schema, + const CoreOptions& core_options) const override; + + Result> NewTableRead( + const std::shared_ptr& path_factory, + const std::shared_ptr& internal_context, + const std::shared_ptr& memory_pool, + const std::shared_ptr& executor) const override; + + CommitScanner::ScanSupplier NewCommitScanSupplier( + const std::shared_ptr& snapshot_manager, + const std::shared_ptr& manifest_list, + const std::shared_ptr& manifest_file, + const std::shared_ptr& table_schema, + const std::shared_ptr& arrow_schema, const CoreOptions& options, + const std::shared_ptr& executor, + const std::shared_ptr& pool) const override; +}; + +} // namespace paimon diff --git a/src/paimon/core/table/file_store.cpp b/src/paimon/core/table/file_store.cpp index e70e85c1d..80b6afb05 100644 --- a/src/paimon/core/table/file_store.cpp +++ b/src/paimon/core/table/file_store.cpp @@ -26,12 +26,18 @@ #include "paimon/catalog/catalog.h" #include "paimon/common/utils/arrow/status_utils.h" +#include "paimon/common/utils/binary_row_partition_computer.h" #include "paimon/core/catalog/version_managed_catalog.h" #include "paimon/core/core_options.h" +#include "paimon/core/index/index_file_handler.h" +#include "paimon/core/manifest/index_manifest_file.h" +#include "paimon/core/manifest/manifest_file.h" +#include "paimon/core/manifest/manifest_list.h" #include "paimon/core/schema/schema_manager.h" #include "paimon/core/schema/table_schema.h" #include "paimon/core/utils/branch_manager.h" #include "paimon/core/utils/file_store_path_factory.h" +#include "paimon/core/utils/index_file_path_factories.h" #include "paimon/core/utils/snapshot_manager.h" #include "paimon/result.h" #include "paimon/status.h" @@ -62,8 +68,13 @@ Result> FileStore::NewPathFactory( const std::shared_ptr& arrow_schema, const CoreOptions& options, const std::string& file_format_identifier, const std::shared_ptr& memory_pool) const { - return MakePathFactory(root_path_, arrow_schema, schema_->PartitionKeys(), options, - file_format_identifier, memory_pool); + PAIMON_ASSIGN_OR_RAISE(std::vector external_paths, options.CreateExternalPaths()); + PAIMON_ASSIGN_OR_RAISE(std::optional global_index_external_path, + options.CreateGlobalIndexExternalPath()); + return FileStorePathFactory::Create( + root_path_, arrow_schema, schema_->PartitionKeys(), options.GetPartitionDefaultName(), + file_format_identifier, options.DataFilePrefix(), options.LegacyPartitionNameEnabled(), + external_paths, global_index_external_path, options.IndexFileInDataFileDir(), memory_pool); } Result> FileStore::NewSnapshotManager( @@ -84,17 +95,55 @@ Result> FileStore::NewSnapshotManager( return snapshot_manager; } -Result> FileStore::MakePathFactory( - const std::string& root_path, const std::shared_ptr& arrow_schema, - const std::vector& partition_keys, const CoreOptions& options, - const std::string& file_format_identifier, const std::shared_ptr& memory_pool) { - PAIMON_ASSIGN_OR_RAISE(std::vector external_paths, options.CreateExternalPaths()); - PAIMON_ASSIGN_OR_RAISE(std::optional global_index_external_path, - options.CreateGlobalIndexExternalPath()); - return FileStorePathFactory::Create( - root_path, arrow_schema, partition_keys, options.GetPartitionDefaultName(), - file_format_identifier, options.DataFilePrefix(), options.LegacyPartitionNameEnabled(), - external_paths, global_index_external_path, options.IndexFileInDataFileDir(), memory_pool); +Result> FileStore::NewManifestList( + const CoreOptions& options, const std::shared_ptr& path_factory, + const std::shared_ptr& memory_pool) const { + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr manifest_list, + ManifestList::Create(options.GetFileSystem(), options.GetManifestFormat(), + options.GetManifestCompression(), path_factory, options.GetCache(), + memory_pool)); + return manifest_list; +} + +Result> FileStore::NewManifestFile( + const CoreOptions& options, const std::shared_ptr& path_factory, + const std::shared_ptr& partition_schema, + const std::shared_ptr& memory_pool) const { + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr manifest_file, + ManifestFile::Create(options.GetFileSystem(), options.GetManifestFormat(), + options.GetManifestCompression(), path_factory, + options.GetManifestTargetFileSize(), memory_pool, options, + partition_schema)); + return manifest_file; +} + +Result> FileStore::NewIndexManifestFile( + const CoreOptions& options, const std::shared_ptr& path_factory, + const std::shared_ptr& memory_pool) const { + return IndexManifestFile::Create(options.GetFileSystem(), options.GetManifestFormat(), + options.GetManifestCompression(), path_factory, + options.GetBucket(), memory_pool, options); +} + +Result> FileStore::NewIndexFileHandler( + const CoreOptions& options, const std::shared_ptr& path_factory, + const std::shared_ptr& memory_pool) const { + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr index_manifest_file, + NewIndexManifestFile(options, path_factory, memory_pool)); + return std::make_unique( + options.GetFileSystem(), std::move(index_manifest_file), + std::make_shared(path_factory), options.DeletionVectorsBitmap64(), + memory_pool); +} + +Result> FileStore::NewPartitionComputer( + const std::shared_ptr& arrow_schema, const CoreOptions& options, + const std::shared_ptr& memory_pool) const { + return BinaryRowPartitionComputer::Create(schema_->PartitionKeys(), arrow_schema, + options.GetPartitionDefaultName(), + options.LegacyPartitionNameEnabled(), memory_pool); } } // namespace paimon diff --git a/src/paimon/core/table/file_store.h b/src/paimon/core/table/file_store.h index ec3bc155c..83c34b1e7 100644 --- a/src/paimon/core/table/file_store.h +++ b/src/paimon/core/table/file_store.h @@ -22,11 +22,9 @@ #include #include #include -#include #include "paimon/catalog/identifier.h" #include "paimon/core/schema/schema_manager.h" -#include "paimon/core/utils/branch_manager.h" #include "paimon/core/utils/file_store_path_factory.h" #include "paimon/core/utils/snapshot_manager.h" #include "paimon/result.h" @@ -38,8 +36,13 @@ class Schema; namespace paimon { +class BinaryRowPartitionComputer; class Catalog; class CoreOptions; +class IndexFileHandler; +class IndexManifestFile; +class ManifestFile; +class ManifestList; class TableSchema; /// The storage a managed table is served through: the file system, root path, schema and branch @@ -86,9 +89,11 @@ class FileStore { } /// The path factory an operator builds its metadata paths through: over this store's root - /// path and the partition keys of its schema. The file format identifier is the caller's to - /// name: a write names the format its level-0 files are written in, which the options may - /// set per level, while a scan, a read and a commit name the table's format. + /// path and the partition keys of its schema - which no schema change can alter, so a read + /// serving a fallback branch's schema still names the same partitions. The file format + /// identifier is the caller's to name: a write names the format its level-0 files are + /// written in, which the options may set per level, while a scan, a read and a commit name + /// the table's format. Result> NewPathFactory( const std::shared_ptr& arrow_schema, const CoreOptions& options, const std::string& file_format_identifier, @@ -105,13 +110,40 @@ class FileStore { const CoreOptions& options, const std::shared_ptr& catalog, const std::optional& catalog_identifier) const; - /// The assembly the factory methods above share: a path factory over a root path and - /// partition keys named by the caller rather than the store's own, which the read path - /// needs - the schema it serves may be a fallback branch's, not the table's. - static Result> MakePathFactory( - const std::string& root_path, const std::shared_ptr& arrow_schema, - const std::vector& partition_keys, const CoreOptions& options, - const std::string& file_format_identifier, const std::shared_ptr& memory_pool); + /// The manifest list a scan or a commit reads the table's manifest entries through, over + /// the caller's path factory: an operator's manifests sit in the paths its path factory + /// names. + Result> NewManifestList( + const CoreOptions& options, const std::shared_ptr& path_factory, + const std::shared_ptr& memory_pool) const; + + /// The manifest file a scan or a commit reads a single manifest's entries through, over + /// the caller's path factory and the partition schema its entries are pruned by. + Result> NewManifestFile( + const CoreOptions& options, const std::shared_ptr& path_factory, + const std::shared_ptr& partition_schema, + const std::shared_ptr& memory_pool) const; + + /// The index manifest file a write, a scan or a commit reads the table's index manifest + /// through, over the caller's path factory. + Result> NewIndexManifestFile( + const CoreOptions& options, const std::shared_ptr& path_factory, + const std::shared_ptr& memory_pool) const; + + /// The index file handler a write's deletion vectors and indexes are maintained through, or + /// a scan reads them through: the index manifest file above and the path factories over the + /// caller's path factory, in one handler. + Result> NewIndexFileHandler( + const CoreOptions& options, const std::shared_ptr& path_factory, + const std::shared_ptr& memory_pool) const; + + /// The partition computer a commit names the partitions it writes with, over the partition + /// keys of this store's schema - which no schema change can alter, so a commit serving a + /// schema read after the table was loaded still names the same partitions - and the + /// caller's arrow schema and options. + Result> NewPartitionComputer( + const std::shared_ptr& arrow_schema, const CoreOptions& options, + const std::shared_ptr& memory_pool) const; private: std::shared_ptr file_system_; diff --git a/src/paimon/core/table/table.cpp b/src/paimon/core/table/file_store_table.cpp similarity index 66% rename from src/paimon/core/table/table.cpp rename to src/paimon/core/table/file_store_table.cpp index cb86e4281..502556990 100644 --- a/src/paimon/core/table/table.cpp +++ b/src/paimon/core/table/file_store_table.cpp @@ -9,17 +9,16 @@ * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ -#include "paimon/catalog/table.h" +#include "paimon/core/table/file_store_table.h" -#include -#include #include #include #include @@ -27,18 +26,14 @@ #include #include -#include "arrow/c/bridge.h" #include "fmt/format.h" #include "paimon/catalog/catalog.h" #include "paimon/catalog/identifier.h" #include "paimon/commit_context.h" -#include "paimon/common/global_index/btree/btree_defs.h" #include "paimon/common/predicate/predicate_validator.h" #include "paimon/common/types/data_field.h" -#include "paimon/common/utils/arrow/status_utils.h" #include "paimon/common/utils/binary_row_partition_computer.h" #include "paimon/common/utils/checked_cast.h" -#include "paimon/common/utils/fields_comparator.h" #include "paimon/common/utils/options_utils.h" #include "paimon/common/utils/path_util.h" #include "paimon/common/utils/string_utils.h" @@ -50,57 +45,36 @@ #include "paimon/core/core_options.h" #include "paimon/core/disk/io_manager.h" #include "paimon/core/index/index_file_handler.h" -#include "paimon/core/index/pk/bucketed_primary_key_index_maintainer.h" #include "paimon/core/index/pk/primary_key_index_definitions.h" -#include "paimon/core/manifest/index_manifest_entry.h" #include "paimon/core/manifest/index_manifest_file.h" -#include "paimon/core/mergetree/compact/lookup_merge_function.h" -#include "paimon/core/mergetree/compact/merge_function.h" -#include "paimon/core/mergetree/compact/reducer_merge_function_wrapper.h" -#include "paimon/core/operation/append_only_file_store_scan.h" -#include "paimon/core/operation/append_only_file_store_write.h" #include "paimon/core/operation/commit/realtime_commit_properties.h" -#include "paimon/core/operation/data_evolution_file_store_scan.h" #include "paimon/core/operation/expire_snapshots.h" #include "paimon/core/operation/file_store_commit_impl.h" #include "paimon/core/operation/file_store_scan.h" #include "paimon/core/operation/internal_read_context.h" -#include "paimon/core/operation/key_value_file_store_scan.h" -#include "paimon/core/operation/key_value_file_store_write.h" -#include "paimon/core/options/merge_engine.h" #include "paimon/core/options/table_type.h" -#include "paimon/core/postpone/postpone_bucket_file_store_write.h" #include "paimon/core/realtime/realtime_context_impl.h" -#include "paimon/core/realtime/realtime_schema_layout.h" #include "paimon/core/schema/schema_manager.h" #include "paimon/core/schema/schema_validation.h" #include "paimon/core/schema/table_schema.h" -#include "paimon/core/table/bucket_mode.h" +#include "paimon/core/snapshot.h" +#include "paimon/core/table/append_only_file_store_table.h" #include "paimon/core/table/file_store.h" -#include "paimon/core/table/file_store_table.h" -#include "paimon/core/table/source/abstract_table_scan.h" -#include "paimon/core/table/source/append_only_split_generator.h" -#include "paimon/core/table/source/append_only_table_read.h" +#include "paimon/core/table/primary_key_file_store_table.h" #include "paimon/core/table/source/data_evolution_batch_scan.h" -#include "paimon/core/table/source/data_evolution_split_generator.h" #include "paimon/core/table/source/data_table_batch_scan.h" #include "paimon/core/table/source/data_table_stream_scan.h" #include "paimon/core/table/source/fallback_table_read.h" -#include "paimon/core/table/source/key_value_table_read.h" -#include "paimon/core/table/source/merge_tree_split_generator.h" #include "paimon/core/table/source/primary_key_index_batch_scan.h" #include "paimon/core/table/source/read_optimized_scan_options.h" #include "paimon/core/table/source/realtime_table_scan.h" #include "paimon/core/table/source/snapshot/snapshot_reader.h" -#include "paimon/core/table/source/split_generator.h" #include "paimon/core/table/system/system_table.h" #include "paimon/core/utils/branch_manager.h" #include "paimon/core/utils/field_mapping.h" #include "paimon/core/utils/file_store_path_factory.h" -#include "paimon/core/utils/index_file_path_factories.h" #include "paimon/core/utils/primary_key_table_utils.h" #include "paimon/core/utils/snapshot_manager.h" -#include "paimon/defs.h" #include "paimon/file_store_commit.h" #include "paimon/file_store_write.h" #include "paimon/format/file_format.h" @@ -122,12 +96,9 @@ class Schema; } // namespace arrow namespace paimon { -class DataSplit; + class Executor; -struct KeyValue; class MemoryPool; -template -class MergeFunctionWrapper; namespace { @@ -170,46 +141,6 @@ Result> TryLoadFormatTable( /*location_carries_paimon_metadata=*/true, options); } -Status RestoreRealtimeCommittedProgress(const std::shared_ptr& realtime_context, - const std::shared_ptr& snapshot_manager, - const CoreOptions& options) { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr realtime_context_impl, - RealtimeContextImpl::Cast(realtime_context)); - PAIMON_RETURN_NOT_OK(realtime_context_impl->CheckUsable()); - PAIMON_ASSIGN_OR_RAISE(std::optional latest_snapshot, - snapshot_manager->LatestSnapshot()); - if (latest_snapshot) { - PAIMON_ASSIGN_OR_RAISE( - RealtimeOffsetMap realtime_committed_offsets, - RealtimeCommitProperties::ReadOffsets(latest_snapshot, options.GetFileSystem())); - PAIMON_RETURN_NOT_OK(realtime_context_impl->AdvanceCommittedProgress( - latest_snapshot->Id(), realtime_committed_offsets, /*committed_data_files=*/{})); - } - return Status::OK(); -} - -Result HasHistoricalPrimaryKeyBTreeDefinition( - const std::shared_ptr& schema_manager, int64_t current_schema_id) { - if (current_schema_id <= TableSchema::FIRST_SCHEMA_ID) { - return false; - } - PAIMON_ASSIGN_OR_RAISE(std::vector schema_ids, schema_manager->ListAllIds()); - for (int64_t schema_id : schema_ids) { - if (schema_id >= current_schema_id) { - continue; - } - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr historical_schema, - schema_manager->ReadSchema(schema_id)); - const auto& historical_options = historical_schema->Options(); - auto option = historical_options.find(Options::PK_BTREE_INDEX_COLUMNS); - if (option != historical_options.end() && - !StringUtils::IsNullOrWhitespaceOnly(option->second)) { - return true; - } - } - return false; -} - /// The tail every `TableOf` ends with: the schema one of them read says what the path names. A /// format table is handed back as the `FormatTable` it is, everything else as the /// `FileStoreTable` a managed table is, carrying the schema for its `NewXxx` to build from. A @@ -230,8 +161,8 @@ Result> TableOfLoadedSchema( // it sits in - the same convention `TryLoadFormatTable` follows above. Identifier table_identifier = identifier.value_or(Identifier(Identifier::kUnknownDatabase, PathUtil::GetName(path))); - return std::make_shared(file_system, path, latest_schema, table_identifier, - uuid, branch, schema_manager); + return FileStoreTable::Create(file_system, path, latest_schema, table_identifier, uuid, branch, + schema_manager); } /// The managed table a context names behind a path: the schema is read through a manager of the @@ -430,8 +361,8 @@ Result> TableOf(const CommitContext& ctx) { MakeBranchCatalogIdentifier(ctx.GetIdentifier(), branch)); // Catalog-managed schemas may be absent from the table directory, but only on the main branch: // a branch publishes its own under `branch/branch-`, which is where the load below and a - // read of the branch both read it, so a catalog answering for no branch schema does not fail - // a commit to one. + // read of the branch both read it, so a catalog answering for no branch schema does not fail a + // commit to one. PAIMON_ASSIGN_OR_RAISE( std::optional catalog_table_schema, CatalogMainBranchSchemaJson(ctx.GetCatalog(), catalog_identifier, branch, "commit")); @@ -527,6 +458,25 @@ FileStoreTable::FileStoreTable(const std::shared_ptr& file_system, std::static_pointer_cast(schema), branch, schema_manager, snapshot_manager)) {} +Result> FileStoreTable::Create( + const std::shared_ptr& file_system, const std::string& root_path, + const std::shared_ptr& schema, const Identifier& identifier, const std::string& uuid, + const std::string& branch, const std::shared_ptr& schema_manager, + const std::shared_ptr& snapshot_manager) { + // The kind the schema's primary keys name, which no schema change can add or drop, so the + // kind the table is built as is the kind every schema its root path ever held describes. A + // caller without a schema gets the append-only kind, whose operators then report that no + // schema was found. + std::shared_ptr table_schema = std::static_pointer_cast(schema); + if (table_schema != nullptr && !table_schema->PrimaryKeys().empty()) { + return std::make_shared(file_system, root_path, schema, + identifier, uuid, branch, schema_manager, + snapshot_manager); + } + return std::make_shared( + file_system, root_path, schema, identifier, uuid, branch, schema_manager, snapshot_manager); +} + Result> FileStoreTable::Copy( const std::map& dynamic_options) const { std::shared_ptr loaded_schema = std::static_pointer_cast(schema_); @@ -541,10 +491,10 @@ Result> FileStoreTable::Copy( loaded_schema->MergeOptions(overridable)); // The managers are handed over, not rebuilt: the copy serves the same root path and branch, // so its operators read the schema cache this table already warmed. - return std::shared_ptr
(std::make_shared( - file_system_, root_path_, std::shared_ptr(std::move(merged_schema)), - identifier_, CatalogUuid().value_or(""), CurrentBranch(), GetSchemaManager(), - GetSnapshotManager())); + return FileStoreTable::Create(file_system_, root_path_, + std::shared_ptr(std::move(merged_schema)), + identifier_, CatalogUuid().value_or(""), CurrentBranch(), + GetSchemaManager(), GetSnapshotManager()); } Result> FileStoreTable::SwitchToBranch(const std::string& branch) const { @@ -552,9 +502,9 @@ Result> FileStoreTable::SwitchToBranch(const std::string& if (target_branch == CurrentBranch()) { // The table already serves this branch: a copy of it - schema and managers shared - is // the same table to the caller. - return std::shared_ptr
(std::make_shared( - file_system_, root_path_, schema_, identifier_, CatalogUuid().value_or(""), - CurrentBranch(), GetSchemaManager(), GetSnapshotManager())); + return FileStoreTable::Create(file_system_, root_path_, schema_, identifier_, + CatalogUuid().value_or(""), CurrentBranch(), + GetSchemaManager(), GetSnapshotManager()); } // The schema of the branch switched to, read from its own directory, which a branch without // one does not have. @@ -572,9 +522,33 @@ Result> FileStoreTable::SwitchToBranch(const std::string& } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr branch_schema, (*latest)->MergeOptions(overridable)); - return std::shared_ptr
(std::make_shared( + return FileStoreTable::Create( file_system_, root_path_, std::shared_ptr(std::move(branch_schema)), - identifier_, CatalogUuid().value_or(""), target_branch, schema_manager)); + identifier_, CatalogUuid().value_or(""), target_branch, schema_manager); +} + +Status FileStoreTable::RestoreRealtimeCommittedProgress( + const std::shared_ptr& realtime_context, + const std::shared_ptr& snapshot_manager, const CoreOptions& options) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr realtime_context_impl, + RealtimeContextImpl::Cast(realtime_context)); + PAIMON_RETURN_NOT_OK(realtime_context_impl->CheckUsable()); + PAIMON_ASSIGN_OR_RAISE(std::optional latest_snapshot, + snapshot_manager->LatestSnapshot()); + if (latest_snapshot) { + PAIMON_ASSIGN_OR_RAISE( + RealtimeOffsetMap realtime_committed_offsets, + RealtimeCommitProperties::ReadOffsets(latest_snapshot, options.GetFileSystem())); + PAIMON_RETURN_NOT_OK(realtime_context_impl->AdvanceCommittedProgress( + latest_snapshot->Id(), realtime_committed_offsets, /*committed_data_files=*/{})); + } + return Status::OK(); +} + +std::unique_ptr FileStoreTable::WithTablePath(std::unique_ptr&& scan, + const std::string& root_path) { + scan->WithTablePath(root_path); + return std::move(scan); } Result> FileStoreTable::NewWrite( @@ -639,209 +613,9 @@ Result> FileStoreTable::NewWrite( RealtimeContextImpl::Cast(ctx->GetRealtimeContext())); PAIMON_RETURN_NOT_OK(realtime_context_impl->CheckUsable()); } - if (schema->PrimaryKeys().empty()) { - // append table - const bool realtime_deduplicate = !options.GetRealtimeDeduplicateKeyFields().empty(); - bool need_dv_maintainer_factory = options.DeletionVectorsEnabled(); - if (options.GetBucket() == -1) { - need_dv_maintainer_factory = false; - ignore_previous_files = true; - } else if (options.GetBucket() <= 0) { - return Status::Invalid( - fmt::format("not support bucket {} in append table", options.GetBucket())); - } - if (ctx->GetRealtimeContext()) { - if (!ctx->GetWriteSchema().empty()) { - return Status::Invalid("real-time append write does not support a custom schema"); - } - if (!realtime_deduplicate) { - PAIMON_RETURN_NOT_OK(RestoreRealtimeCommittedProgress(ctx->GetRealtimeContext(), - snapshot_manager, options)); - } - } else if (realtime_deduplicate) { - return Status::Invalid("real-time deduplicate write requires a real-time context"); - } - std::shared_ptr write_schema = arrow_schema; - const auto& write_field_names = ctx->GetWriteSchema(); - if (!write_field_names.empty()) { - arrow::FieldVector write_fields; - write_fields.reserve(write_field_names.size()); - for (const auto& field_name : write_field_names) { - auto field = arrow_schema->GetFieldByName(field_name); - if (!field) { - // TODO(xinyu.lxy): support _ROW_ID and _SEQUENCE_NUMBER - return Status::Invalid( - fmt::format("write field {} does not exist in table schema", field_name)); - } - write_fields.push_back(field); - } - write_schema = arrow::schema(write_fields); - } - - std::shared_ptr realtime_schema_layout; - if (ctx->GetRealtimeContext()) { - const RealtimeStoreMode mode = realtime_deduplicate ? RealtimeStoreMode::DEDUPLICATE - : RealtimeStoreMode::APPEND_ONLY; - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr layout, - RealtimeSchemaLayout::Create(mode, write_schema)); - realtime_schema_layout = std::move(layout); - } - - std::shared_ptr dv_maintainer_factory; - if (need_dv_maintainer_factory) { - PAIMON_ASSIGN_OR_RAISE( - std::unique_ptr index_manifest_file, - IndexManifestFile::Create(options.GetFileSystem(), options.GetManifestFormat(), - options.GetManifestCompression(), file_store_path_factory, - options.GetBucket(), ctx->GetMemoryPool(), options)); - auto index_file_handler = std::make_shared( - options.GetFileSystem(), std::move(index_manifest_file), - std::make_shared(file_store_path_factory), - options.DeletionVectorsBitmap64(), ctx->GetMemoryPool()); - dv_maintainer_factory = - std::make_shared(index_file_handler); - } - - auto file_store_write = std::make_unique( - file_store_path_factory, snapshot_manager, GetSchemaManager(), ctx->GetCommitUser(), - root_path_, schema, arrow_schema, write_schema, realtime_schema_layout, - partition_schema, dv_maintainer_factory, io_manager, options, ignore_previous_files, - ctx->IsStreamingMode(), ctx->IgnoreNumBucketCheck(), ctx->GetRealtimeContext(), - ctx->GetExecutor(), ctx->GetMemoryPool()); - if (ctx->GetRealtimeContext() && realtime_deduplicate) { - PAIMON_ASSIGN_OR_RAISE(std::optional latest_snapshot, - snapshot_manager->LatestSnapshot()); - if (latest_snapshot) { - PAIMON_RETURN_NOT_OK( - file_store_write->RefreshCommittedSnapshot(latest_snapshot->Id())); - } - } - return std::unique_ptr(std::move(file_store_write)); - } else { - // pk table - if (ctx->GetRealtimeContext()) { - PAIMON_RETURN_NOT_OK(PrimaryKeyTableUtils::ValidateRealtimeOptions(options, *schema)); - if (ignore_previous_files) { - return Status::NotImplemented( - "PK realtime requires restore from the latest snapshot"); - } - if (!ctx->GetWriteSchema().empty()) { - return Status::NotImplemented("PK realtime does not support a custom write schema"); - } - PAIMON_RETURN_NOT_OK(RestoreRealtimeCommittedProgress(ctx->GetRealtimeContext(), - snapshot_manager, options)); - } - if (options.GetBucket() == BucketModeDefine::POSTPONE_BUCKET) { - return PostponeBucketFileStoreWrite::Create( - snapshot_manager, GetSchemaManager(), ctx->GetCommitUser(), root_path_, schema, - arrow_schema, partition_schema, io_manager, options, ctx->IsStreamingMode(), - ctx->IgnoreNumBucketCheck(), ctx->GetWriteId(), - ctx->GetFileSystemSchemeToIdentifierMap(), ctx->GetExecutor(), ctx->GetMemoryPool(), - file_system_); - } - if (options.GetBucket() <= 0) { - return Status::Invalid( - fmt::format("not support bucket {} in key value table", options.GetBucket())); - } - PAIMON_ASSIGN_OR_RAISE(std::vector trimmed_primary_keys, - schema->TrimmedPrimaryKeys()); - PAIMON_ASSIGN_OR_RAISE(std::vector trimmed_primary_key_fields, - schema->GetFields(trimmed_primary_keys)); - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr key_comparator, - FieldsComparator::Create(trimmed_primary_key_fields, - options.SequenceFieldSortOrderIsAscending())); - auto primary_keys = schema->PrimaryKeys(); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr merge_function, - PrimaryKeyTableUtils::CreateMergeFunction( - arrow_schema, primary_keys, options, ctx->GetMemoryPool())); - if (options.NeedLookup() && options.GetMergeEngine() != MergeEngine::FIRST_ROW) { - // don't wrap first row, it is already OK - merge_function = std::make_unique(std::move(merge_function)); - } - std::shared_ptr> merge_function_wrapper = - std::make_shared(std::move(merge_function)); - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr sequence_fields_comparator, - PrimaryKeyTableUtils::CreateSequenceFieldsComparator(schema->Fields(), options)); - - std::shared_ptr dv_maintainer_factory; - PAIMON_ASSIGN_OR_RAISE(PrimaryKeyIndexDefinitions primary_key_index_definitions, - PrimaryKeyIndexDefinitions::Create(*schema)); - bool has_btree_index = false; - for (const PrimaryKeyIndexDefinition& definition : - primary_key_index_definitions.Definitions()) { - if (definition.GetFamily() == PrimaryKeyIndexDefinition::Family::BTREE) { - has_btree_index = true; - break; - } - } - std::optional latest_snapshot; - bool has_existing_index_manifest = false; - if (!has_btree_index) { - PAIMON_ASSIGN_OR_RAISE( - bool has_historical_btree_index, - HasHistoricalPrimaryKeyBTreeDefinition(GetSchemaManager(), schema->Id())); - if (has_historical_btree_index) { - PAIMON_ASSIGN_OR_RAISE(latest_snapshot, snapshot_manager->LatestSnapshot()); - has_existing_index_manifest = - latest_snapshot.has_value() && latest_snapshot->IndexManifest().has_value(); - } - } - std::shared_ptr index_file_handler; - if (options.DeletionVectorsEnabled() || has_btree_index || has_existing_index_manifest) { - PAIMON_ASSIGN_OR_RAISE( - std::unique_ptr index_manifest_file, - IndexManifestFile::Create(options.GetFileSystem(), options.GetManifestFormat(), - options.GetManifestCompression(), file_store_path_factory, - options.GetBucket(), ctx->GetMemoryPool(), options)); - index_file_handler = std::make_shared( - options.GetFileSystem(), std::move(index_manifest_file), - std::make_shared(file_store_path_factory), - options.DeletionVectorsBitmap64(), ctx->GetMemoryPool()); - } - bool has_restored_btree_index = false; - if (!has_btree_index && has_existing_index_manifest) { - PAIMON_ASSIGN_OR_RAISE( - std::vector restored_btree_entries, - index_file_handler->Scan( - latest_snapshot.value(), [](const IndexManifestEntry& entry) -> bool { - return entry.index_file->IndexType() == BtreeDefs::kIdentifier && - IndexFileHandler::IsPrimaryKeySourceIndex(*entry.index_file); - })); - has_restored_btree_index = !restored_btree_entries.empty(); - } - if (options.DeletionVectorsEnabled()) { - dv_maintainer_factory = - std::make_shared(index_file_handler); - } - std::shared_ptr - primary_key_index_maintainer_factory; - if (has_btree_index || has_restored_btree_index) { - primary_key_index_maintainer_factory = - std::make_shared( - root_path_, CurrentBranch(), schema, - primary_key_index_definitions.Definitions(), file_store_path_factory, - index_file_handler, options, io_manager, ctx->EnableMultiThreadSpill(), - ctx->GetExecutor(), ctx->GetMemoryPool()); - } - - std::shared_ptr realtime_schema_layout; - if (ctx->GetRealtimeContext()) { - PAIMON_ASSIGN_OR_RAISE( - std::unique_ptr layout, - RealtimeSchemaLayout::Create(RealtimeStoreMode::PRIMARY_KEY, arrow_schema)); - realtime_schema_layout = std::move(layout); - } - - return std::make_unique( - file_store_path_factory, snapshot_manager, GetSchemaManager(), ctx->GetCommitUser(), - root_path_, schema, arrow_schema, realtime_schema_layout, partition_schema, - dv_maintainer_factory, primary_key_index_maintainer_factory, io_manager, key_comparator, - sequence_fields_comparator, merge_function_wrapper, options, ignore_previous_files, - ctx->IsStreamingMode(), ctx->IgnoreNumBucketCheck(), ctx->EnableMultiThreadSpill(), - ctx->GetRealtimeContext(), ctx->GetExecutor(), ctx->GetMemoryPool()); - } + return CreateWrite(*ctx, schema, arrow_schema, partition_schema, options, + file_store_path_factory, snapshot_manager, io_manager, + ignore_previous_files); } Result> FileStoreWrite::Create(std::unique_ptr ctx) { @@ -853,110 +627,6 @@ Result> FileStoreWrite::Create(std::unique_ptr> CreateFileStoreScan( - const std::shared_ptr& path_factory, - const std::shared_ptr& arrow_schema, - const std::shared_ptr& table_schema, const CoreOptions& core_options, - const std::shared_ptr& snapshot_manager, - const std::shared_ptr& schema_manager, - const std::shared_ptr& executor, const std::shared_ptr& memory_pool, - const ScanContext* context, const std::string& root_path) { - auto fs = core_options.GetFileSystem(); - auto manifest_file_format = core_options.GetManifestFormat(); - // TODO(liancheng.lsz): support fallback branch in scan - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr manifest_list, - ManifestList::Create(fs, manifest_file_format, core_options.GetManifestCompression(), - path_factory, core_options.GetCache(), memory_pool)); - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr partition_schema, - FieldMapping::GetPartitionSchema(arrow_schema, table_schema->PartitionKeys())); - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr manifest_file, - ManifestFile::Create(fs, manifest_file_format, core_options.GetManifestCompression(), - path_factory, core_options.GetManifestTargetFileSize(), - memory_pool, core_options, partition_schema)); - std::unique_ptr scan; - if (table_schema->PrimaryKeys().empty()) { - if (core_options.DataEvolutionEnabled()) { - PAIMON_ASSIGN_OR_RAISE( - scan, DataEvolutionFileStoreScan::Create( - snapshot_manager, schema_manager, manifest_list, manifest_file, - table_schema, arrow_schema, context->GetScanFilters(), core_options, - executor, memory_pool)); - } else { - PAIMON_ASSIGN_OR_RAISE( - scan, AppendOnlyFileStoreScan::Create( - snapshot_manager, schema_manager, manifest_list, manifest_file, - table_schema, arrow_schema, context->GetScanFilters(), core_options, - executor, memory_pool)); - } - } else { - PAIMON_ASSIGN_OR_RAISE( - scan, KeyValueFileStoreScan::Create(snapshot_manager, schema_manager, manifest_list, - manifest_file, table_schema, arrow_schema, - context->GetScanFilters(), core_options, - executor, memory_pool)); - } - return WithTablePath(std::move(scan), root_path); - } - - static std::unique_ptr WithTablePath(std::unique_ptr&& scan, - const std::string& root_path) { - scan->WithTablePath(root_path); - return std::move(scan); - } - - static Result> CreateSplitGenerator( - const std::shared_ptr& table_schema, const CoreOptions& core_options, - const ScanContext* context) { - auto source_split_target_size = core_options.GetSourceSplitTargetSize(); - auto source_split_open_file_cost = core_options.GetSourceSplitOpenFileCost(); - if (table_schema->PrimaryKeys().empty()) { - if (core_options.DataEvolutionEnabled()) { - return std::make_unique( - source_split_target_size, source_split_open_file_cost, - core_options.BlobSplitByFileSize()); - } - BucketMode bucket_mode = (core_options.GetBucket() == -1 ? BucketMode::BUCKET_UNAWARE - : BucketMode::HASH_FIXED); - return std::make_unique( - source_split_target_size, source_split_open_file_cost, bucket_mode); - } else { - // TODO(liancheng.lsz): support evolution - PAIMON_ASSIGN_OR_RAISE(std::vector trimmed_primary_keys, - table_schema->TrimmedPrimaryKeys()); - PAIMON_ASSIGN_OR_RAISE(std::vector trimmed_pk_fields, - table_schema->GetFields(trimmed_primary_keys)); - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr key_comparator, - FieldsComparator::Create(trimmed_pk_fields, /*is_ascending_order=*/true)); - return std::make_unique( - source_split_target_size, source_split_open_file_cost, - core_options.DeletionVectorsEnabled(), core_options.GetMergeEngine(), - key_comparator); - } - } - - static Result> CreateIndexFileHandler( - const CoreOptions& core_options, const std::shared_ptr& path_factory, - const std::shared_ptr& memory_pool) { - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr index_manifest_file, - IndexManifestFile::Create( - core_options.GetFileSystem(), core_options.GetManifestFormat(), - core_options.GetManifestCompression(), path_factory, - core_options.GetBucket(), memory_pool, core_options)); - return std::make_unique( - core_options.GetFileSystem(), std::move(index_manifest_file), - std::make_shared(path_factory), - core_options.DeletionVectorsBitmap64(), memory_pool); - } -}; - Status ValidateRealtimeScan(const TableSchema& table_schema, const CoreOptions& core_options, const ScanContext& context, bool read_optimized) { if (!context.GetRealtimeContext()) { @@ -1064,18 +734,26 @@ Result> FileStoreTable::NewScan( core_options.GetFileFormat()->Identifier(), context->GetMemoryPool())); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr manifest_list, + store_->NewManifestList(core_options, path_factory, context->GetMemoryPool())); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr partition_schema, + FieldMapping::GetPartitionSchema(arrow_schema, table_schema->PartitionKeys())); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr manifest_file, + store_->NewManifestFile(core_options, path_factory, partition_schema, + context->GetMemoryPool())); + // The managers the table carries, whose schema cache the dispatch already warmed. - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr file_store_scan, - TableScanImpl::CreateFileStoreScan( - path_factory, arrow_schema, table_schema, core_options, - GetSnapshotManager(), GetSchemaManager(), context->GetExecutor(), - context->GetMemoryPool(), context.get(), root_path_)); PAIMON_ASSIGN_OR_RAISE( - std::unique_ptr split_generator, - TableScanImpl::CreateSplitGenerator(table_schema, core_options, context.get())); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr index_file_handler, - TableScanImpl::CreateIndexFileHandler(core_options, path_factory, - context->GetMemoryPool())); + std::shared_ptr file_store_scan, + NewFileStoreScan(manifest_list, manifest_file, arrow_schema, table_schema, core_options, + context->GetExecutor(), context->GetMemoryPool(), context.get())); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr split_generator, + NewSplitGenerator(table_schema, core_options)); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr index_file_handler, + store_->NewIndexFileHandler(core_options, path_factory, context->GetMemoryPool())); auto snapshot_reader = std::make_shared( file_store_scan, path_factory, std::move(split_generator), std::move(index_file_handler)); const bool pk_table = !table_schema->PrimaryKeys().empty(); @@ -1171,9 +849,12 @@ Result> CreateInternalReadContext( return InternalReadContext::Create(context, table_schema, options, file_system, &root_path); } -Result> CreateTableRead( +} // namespace + +Result> FileStoreTable::CreateTableRead( const std::shared_ptr& internal_context, - const std::shared_ptr& memory_pool, const std::shared_ptr& executor) { + const std::shared_ptr& memory_pool, + const std::shared_ptr& executor) const { if (PredicateUtils::ContainsFullTextMatch(internal_context->GetPredicate())) { return Status::NotImplemented("FullTextMatch predicate execution is not implemented yet"); } @@ -1189,19 +870,11 @@ Result> CreateTableRead( auto arrow_schema = DataField::ConvertDataFieldsToArrowSchema(table_schema->Fields()); PAIMON_ASSIGN_OR_RAISE( std::shared_ptr path_factory, - FileStore::MakePathFactory(internal_context->GetPath(), arrow_schema, - table_schema->PartitionKeys(), core_options, - core_options.GetFileFormat()->Identifier(), memory_pool)); - - if (internal_context->GetPrimaryKeys().empty()) { - return std::make_unique(path_factory, internal_context, memory_pool, - executor); - } - return KeyValueTableRead::Create(path_factory, internal_context, memory_pool, executor); + store_->NewPathFactory(arrow_schema, core_options, + core_options.GetFileFormat()->Identifier(), memory_pool)); + return NewTableRead(path_factory, internal_context, memory_pool, executor); } -} // namespace - Result> FileStoreTable::NewRead( std::unique_ptr context) const { std::shared_ptr read_context = std::move(context); @@ -1258,46 +931,6 @@ Result> TableRead::Create(std::unique_ptr& snapshot_manager, - const std::shared_ptr& schema_manager, - const std::shared_ptr& manifest_list, - const std::shared_ptr& manifest_file, - const std::shared_ptr& table_schema, - const std::shared_ptr& arrow_schema, const CoreOptions& options, - const std::shared_ptr& executor, const std::shared_ptr& pool) { - return [snapshot_manager, schema_manager, manifest_list, manifest_file, table_schema, - arrow_schema, options, executor, pool](const std::shared_ptr& scan_filter) - -> Result> { - PAIMON_ASSIGN_OR_RAISE( - std::unique_ptr scan, - AppendOnlyFileStoreScan::Create(snapshot_manager, schema_manager, manifest_list, - manifest_file, table_schema, arrow_schema, scan_filter, - options, executor, pool)); - return std::unique_ptr(std::move(scan)); - }; -} - -CommitScanner::ScanSupplier CreatePkScanSupplier( - const std::shared_ptr& snapshot_manager, - const std::shared_ptr& schema_manager, - const std::shared_ptr& manifest_list, - const std::shared_ptr& manifest_file, - const std::shared_ptr& table_schema, - const std::shared_ptr& arrow_schema, const CoreOptions& options, - const std::shared_ptr& executor, const std::shared_ptr& pool) { - return [snapshot_manager, schema_manager, manifest_list, manifest_file, table_schema, - arrow_schema, options, executor, pool](const std::shared_ptr& scan_filter) - -> Result> { - PAIMON_ASSIGN_OR_RAISE( - std::unique_ptr scan, - KeyValueFileStoreScan::Create(snapshot_manager, schema_manager, manifest_list, - manifest_file, table_schema, arrow_schema, scan_filter, - options, executor, pool)); - return std::unique_ptr(std::move(scan)); - }; -} - bool IsCatalogCommit(const CommitContext& ctx) { return AsVersionManaged(ctx.GetCatalog()) != nullptr; } @@ -1394,11 +1027,9 @@ Result> FileStoreTable::NewCommit( return Status::NotImplemented( "commit operation does not support object store file system for now"); } - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr partition_computer, - BinaryRowPartitionComputer::Create(schema->PartitionKeys(), arrow_schema, - options.GetPartitionDefaultName(), - options.LegacyPartitionNameEnabled(), - ctx->GetMemoryPool())); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr partition_computer, + store_->NewPartitionComputer(arrow_schema, options, ctx->GetMemoryPool())); PAIMON_ASSIGN_OR_RAISE( std::shared_ptr path_factory, store_->NewPathFactory(arrow_schema, options, options.GetFileFormat()->Identifier(), @@ -1407,41 +1038,26 @@ Result> FileStoreTable::NewCommit( PAIMON_ASSIGN_OR_RAISE( std::shared_ptr snapshot_manager, store_->NewSnapshotManager(options, ctx->GetCatalog(), catalog_identifier)); - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr manifest_list, - ManifestList::Create(options.GetFileSystem(), options.GetManifestFormat(), - options.GetManifestCompression(), path_factory, options.GetCache(), - ctx->GetMemoryPool())); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr manifest_list, + store_->NewManifestList(options, path_factory, ctx->GetMemoryPool())); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr partition_schema, FieldMapping::GetPartitionSchema(arrow_schema, schema->PartitionKeys())); PAIMON_ASSIGN_OR_RAISE( std::shared_ptr manifest_file, - ManifestFile::Create(options.GetFileSystem(), options.GetManifestFormat(), - options.GetManifestCompression(), path_factory, - options.GetManifestTargetFileSize(), ctx->GetMemoryPool(), options, - partition_schema)); + store_->NewManifestFile(options, path_factory, partition_schema, ctx->GetMemoryPool())); PAIMON_ASSIGN_OR_RAISE( std::shared_ptr index_manifest_file, - IndexManifestFile::Create(options.GetFileSystem(), options.GetManifestFormat(), - options.GetManifestCompression(), path_factory, - options.GetBucket(), ctx->GetMemoryPool(), options)); + store_->NewIndexManifestFile(options, path_factory, ctx->GetMemoryPool())); auto expire_snapshots = std::make_shared( snapshot_manager, path_factory, manifest_list, manifest_file, index_manifest_file, options.GetFileSystem(), options.GetExpireConfig(), options.RealtimeEnabled(), ctx->GetExecutor()); - CommitScanner::ScanSupplier scan_supplier; - if (schema->PrimaryKeys().empty()) { - scan_supplier = CreateAppendScanSupplier(snapshot_manager, GetSchemaManager(), - manifest_list, manifest_file, schema, arrow_schema, - options, ctx->GetExecutor(), ctx->GetMemoryPool()); - } else { - scan_supplier = CreatePkScanSupplier(snapshot_manager, GetSchemaManager(), manifest_list, - manifest_file, schema, arrow_schema, options, - ctx->GetExecutor(), ctx->GetMemoryPool()); - } + CommitScanner::ScanSupplier scan_supplier = + NewCommitScanSupplier(snapshot_manager, manifest_list, manifest_file, schema, arrow_schema, + options, ctx->GetExecutor(), ctx->GetMemoryPool()); PAIMON_ASSIGN_OR_RAISE( std::shared_ptr snapshot_commit, diff --git a/src/paimon/core/table/file_store_table.h b/src/paimon/core/table/file_store_table.h index 51c7c0b95..00959d257 100644 --- a/src/paimon/core/table/file_store_table.h +++ b/src/paimon/core/table/file_store_table.h @@ -25,6 +25,7 @@ #include "paimon/catalog/identifier.h" #include "paimon/catalog/table.h" +#include "paimon/core/operation/commit/commit_scanner.h" #include "paimon/core/schema/schema_manager.h" #include "paimon/core/table/file_store.h" #include "paimon/core/utils/branch_manager.h" @@ -34,26 +35,42 @@ namespace paimon { +class InternalReadContext; +class IOManager; +class ManifestFile; +class ManifestList; +class RealtimeContext; class Schema; +class SplitGenerator; /// The managed table a catalog hands out: one with snapshots and manifests, which the schema under /// its root path describes. A format table - a directory of plain data files - is a `FormatTable`, /// the other kind of `Table`. /// +/// This is the flow the four operators share, Java Paimon's `AbstractFileStoreTable`: what the +/// table was constructed with - the schema, file system, root path, branch and managers - decides +/// what every operator serves, so it is read here once and handed down. The schema's primary keys +/// decide the kind of managed table it is - `AppendOnlyFileStoreTable` or +/// `PrimaryKeyFileStoreTable`, the `Create` factory below - and that kind builds the pieces the +/// flows hand down: the write itself, the scans, the read and the split generator. +/// /// Its `NewScan`/`NewRead`/`NewWrite`/`NewCommit` build the operators here, from the context plus -/// the schema, file system, root path, branch and managers the table was constructed with, so -/// they serve the table as it was loaded. The +/// the state the table was constructed with, so they serve the table as it was loaded. The /// context-only entry points - `TableScan::Create()`, `TableRead::Create()`, /// `FileStoreWrite::Create()` and `FileStoreCommit::Create()` - load the table the context names /// and hand the context to that table's `NewXxx`, which is how a context built without a table /// reaches the same operators. class FileStoreTable : public Table { public: - /// The constructor a catalog builds a managed table through, so a `shared_ptr` can hold one; - /// the base constructor is protected, which an inherited constructor would stay. - /// @param file_system File system the table's data is accessed through. - /// @param root_path Root path of the table in the file system. - /// @param uuid Metastore UUID, or empty if unavailable. + /// The managed table the schema describes, built as the kind its primary keys name: an + /// `AppendOnlyFileStoreTable` when there are none, a `PrimaryKeyFileStoreTable` when there + /// are, the way Java Paimon's `FileStoreTableFactory` does. Primary keys cannot be added or + /// dropped by a schema change, so the kind the table was built as is the kind every schema + /// its root path ever held describes. + /// @param schema The schema the table was loaded with, whose primary keys name the kind and + /// whose options the operators serve. It is null when the caller had no schema, and + /// the table is built as the append-only kind - its operators then report that no + /// schema was found. /// @param branch The branch the table serves, which the managers below read and write on. /// @param schema_manager A schema manager to carry instead of building one - the one a /// context dispatch read the schema through, whose cache the operators then serve. @@ -61,12 +78,13 @@ class FileStoreTable : public Table { /// `Copy` hands its table's over so both share one. /// A null manager is built over `root_path` on `branch`, so the operators always read /// through the managers the table carries. - FileStoreTable(const std::shared_ptr& file_system, const std::string& root_path, - const std::shared_ptr& schema, const Identifier& identifier, - const std::string& uuid = "", - const std::string& branch = BranchManager::DEFAULT_MAIN_BRANCH, - const std::shared_ptr& schema_manager = nullptr, - const std::shared_ptr& snapshot_manager = nullptr); + static Result> Create( + const std::shared_ptr& file_system, const std::string& root_path, + const std::shared_ptr& schema, const Identifier& identifier, + const std::string& uuid = "", + const std::string& branch = BranchManager::DEFAULT_MAIN_BRANCH, + const std::shared_ptr& schema_manager = nullptr, + const std::shared_ptr& snapshot_manager = nullptr); /// The branch this table serves, as resolved when the table was built: the schema the table /// was loaded with came from it, and its managers point at it. @@ -120,6 +138,86 @@ class FileStoreTable : public Table { std::unique_ptr context) const override; private: + /// The read `NewRead` hands back, from the internal context it read the schema into: the + /// path factory comes from the store - a fallback branch's schema carries the table's + /// partition keys, which no schema change can alter - and the read itself is the kind's, + /// `NewTableRead`. + Result> CreateTableRead( + const std::shared_ptr& internal_context, + const std::shared_ptr& memory_pool, + const std::shared_ptr& executor) const; + + protected: + /// The base constructor the kinds of managed table build through, `Create` above. The + /// parameters are the ones it takes; see there for what each one is. + FileStoreTable(const std::shared_ptr& file_system, const std::string& root_path, + const std::shared_ptr& schema, const Identifier& identifier, + const std::string& uuid, const std::string& branch, + const std::shared_ptr& schema_manager, + const std::shared_ptr& snapshot_manager); + + /// The committed progress a real-time write restores, from the latest snapshot the snapshot + /// manager reads: where the write picks up after, which both kinds of real-time write ask + /// for the same way. + static Status RestoreRealtimeCommittedProgress( + const std::shared_ptr& realtime_context, + const std::shared_ptr& snapshot_manager, const CoreOptions& options); + + /// The scan the table's root path names, which the kinds of file store scan read from: the + /// kinds hand their scans back through it. + static std::unique_ptr WithTablePath(std::unique_ptr&& scan, + const std::string& root_path); + + /// The write of this kind of table, which `NewWrite` hands the shared flow's pieces to: the + /// kind's writer, built over the path factory, snapshot manager and options that flow + /// resolved. The branch the write serves is the table's, `CurrentBranch()`. + /// @param io_manager The spill manager the context's temp directory names, or null when the + /// context named none. + /// @param ignore_previous_files Whether the write starts over rather than appending, which + /// the flow resolved with this kind's own rules. + virtual Result> CreateWrite( + const WriteContext& ctx, const std::shared_ptr& schema, + const std::shared_ptr& arrow_schema, + const std::shared_ptr& partition_schema, const CoreOptions& options, + const std::shared_ptr& path_factory, + const std::shared_ptr& snapshot_manager, + const std::shared_ptr& io_manager, bool ignore_previous_files) const = 0; + + /// The scan of this kind of table, which `NewScan` hands the manifest list and file it built + /// to. The snapshot and schema managers are the table's own, which the scan serves. + virtual Result> NewFileStoreScan( + const std::shared_ptr& manifest_list, + const std::shared_ptr& manifest_file, + const std::shared_ptr& arrow_schema, + const std::shared_ptr& table_schema, const CoreOptions& core_options, + const std::shared_ptr& executor, const std::shared_ptr& memory_pool, + const ScanContext* context) const = 0; + + /// The split generator of this kind of table, Java Paimon's `splitGenerator()`. + virtual Result> NewSplitGenerator( + const std::shared_ptr& table_schema, + const CoreOptions& core_options) const = 0; + + /// The read of this kind of table, which `NewRead` hands the internal context it read the + /// schema into and the path factory it built to. + virtual Result> NewTableRead( + const std::shared_ptr& path_factory, + const std::shared_ptr& internal_context, + const std::shared_ptr& memory_pool, + const std::shared_ptr& executor) const = 0; + + /// The manifest entry scan supplier of this kind of table, which `NewCommit` hands the + /// manifest list and file it built to. The schema manager the scans it hands out read through + /// is the table's own. + virtual CommitScanner::ScanSupplier NewCommitScanSupplier( + const std::shared_ptr& snapshot_manager, + const std::shared_ptr& manifest_list, + const std::shared_ptr& manifest_file, + const std::shared_ptr& table_schema, + const std::shared_ptr& arrow_schema, const CoreOptions& options, + const std::shared_ptr& executor, + const std::shared_ptr& pool) const = 0; + /// The storage this table serves: the file system, root path, schema, branch and managers it /// was loaded with, and the factories that build the pieces every operator opens with. std::shared_ptr store_; diff --git a/src/paimon/core/table/primary_key_file_store_table.cpp b/src/paimon/core/table/primary_key_file_store_table.cpp new file mode 100644 index 000000000..564e015e3 --- /dev/null +++ b/src/paimon/core/table/primary_key_file_store_table.cpp @@ -0,0 +1,291 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/table/primary_key_file_store_table.h" + +#include +#include +#include +#include +#include +#include + +#include "fmt/format.h" +#include "paimon/common/global_index/btree/btree_defs.h" +#include "paimon/common/types/data_field.h" +#include "paimon/common/utils/fields_comparator.h" +#include "paimon/common/utils/string_utils.h" +#include "paimon/core/core_options.h" +#include "paimon/core/deletionvectors/bucketed_dv_maintainer.h" +#include "paimon/core/index/index_file_handler.h" +#include "paimon/core/index/pk/bucketed_primary_key_index_maintainer.h" +#include "paimon/core/index/pk/primary_key_index_definitions.h" +#include "paimon/core/manifest/index_manifest_entry.h" +#include "paimon/core/mergetree/compact/lookup_merge_function.h" +#include "paimon/core/mergetree/compact/merge_function.h" +#include "paimon/core/mergetree/compact/reducer_merge_function_wrapper.h" +#include "paimon/core/operation/commit/commit_scanner.h" +#include "paimon/core/operation/file_store_scan.h" +#include "paimon/core/operation/key_value_file_store_scan.h" +#include "paimon/core/operation/key_value_file_store_write.h" +#include "paimon/core/options/merge_engine.h" +#include "paimon/core/postpone/postpone_bucket_file_store_write.h" +#include "paimon/core/realtime/realtime_schema_layout.h" +#include "paimon/core/schema/schema_manager.h" +#include "paimon/core/schema/table_schema.h" +#include "paimon/core/snapshot.h" +#include "paimon/core/table/bucket_mode.h" +#include "paimon/core/table/file_store.h" +#include "paimon/core/table/source/key_value_table_read.h" +#include "paimon/core/table/source/merge_tree_split_generator.h" +#include "paimon/core/table/source/split_generator.h" +#include "paimon/core/utils/primary_key_table_utils.h" +#include "paimon/core/utils/snapshot_manager.h" +#include "paimon/result.h" +#include "paimon/status.h" +#include "paimon/write_context.h" + +namespace paimon { + +struct KeyValue; +template +class MergeFunctionWrapper; + +namespace { + +/// Whether a schema older than `current_schema_id` carried a primary key b-tree index, which a +/// write has to keep maintaining: the index files an earlier schema wrote are still there to be +/// read, and dropping the definition now would leave them orphaned. +Result HasHistoricalPrimaryKeyBTreeDefinition( + const std::shared_ptr& schema_manager, int64_t current_schema_id) { + if (current_schema_id <= TableSchema::FIRST_SCHEMA_ID) { + return false; + } + PAIMON_ASSIGN_OR_RAISE(std::vector schema_ids, schema_manager->ListAllIds()); + for (int64_t schema_id : schema_ids) { + if (schema_id >= current_schema_id) { + continue; + } + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr historical_schema, + schema_manager->ReadSchema(schema_id)); + const auto& historical_options = historical_schema->Options(); + auto option = historical_options.find(Options::PK_BTREE_INDEX_COLUMNS); + if (option != historical_options.end() && + !StringUtils::IsNullOrWhitespaceOnly(option->second)) { + return true; + } + } + return false; +} + +} // namespace + +PrimaryKeyFileStoreTable::PrimaryKeyFileStoreTable( + const std::shared_ptr& file_system, const std::string& root_path, + const std::shared_ptr& schema, const Identifier& identifier, const std::string& uuid, + const std::string& branch, const std::shared_ptr& schema_manager, + const std::shared_ptr& snapshot_manager) + : FileStoreTable(file_system, root_path, schema, identifier, uuid, branch, schema_manager, + snapshot_manager) {} + +Result> PrimaryKeyFileStoreTable::CreateWrite( + const WriteContext& ctx, const std::shared_ptr& schema, + const std::shared_ptr& arrow_schema, + const std::shared_ptr& partition_schema, const CoreOptions& options, + const std::shared_ptr& path_factory, + const std::shared_ptr& snapshot_manager, + const std::shared_ptr& io_manager, bool ignore_previous_files) const { + // pk table + if (ctx.GetRealtimeContext()) { + PAIMON_RETURN_NOT_OK(PrimaryKeyTableUtils::ValidateRealtimeOptions(options, *schema)); + if (ignore_previous_files) { + return Status::NotImplemented("PK realtime requires restore from the latest snapshot"); + } + if (!ctx.GetWriteSchema().empty()) { + return Status::NotImplemented("PK realtime does not support a custom write schema"); + } + PAIMON_RETURN_NOT_OK( + RestoreRealtimeCommittedProgress(ctx.GetRealtimeContext(), snapshot_manager, options)); + } + if (options.GetBucket() == BucketModeDefine::POSTPONE_BUCKET) { + return PostponeBucketFileStoreWrite::Create( + snapshot_manager, GetSchemaManager(), ctx.GetCommitUser(), root_path_, schema, + arrow_schema, partition_schema, io_manager, options, ctx.IsStreamingMode(), + ctx.IgnoreNumBucketCheck(), ctx.GetWriteId(), ctx.GetFileSystemSchemeToIdentifierMap(), + ctx.GetExecutor(), ctx.GetMemoryPool(), file_system_); + } + if (options.GetBucket() <= 0) { + return Status::Invalid( + fmt::format("not support bucket {} in key value table", options.GetBucket())); + } + PAIMON_ASSIGN_OR_RAISE(std::vector trimmed_primary_keys, + schema->TrimmedPrimaryKeys()); + PAIMON_ASSIGN_OR_RAISE(std::vector trimmed_primary_key_fields, + schema->GetFields(trimmed_primary_keys)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr key_comparator, + FieldsComparator::Create(trimmed_primary_key_fields, + options.SequenceFieldSortOrderIsAscending())); + auto primary_keys = schema->PrimaryKeys(); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr merge_function, + PrimaryKeyTableUtils::CreateMergeFunction(arrow_schema, primary_keys, + options, ctx.GetMemoryPool())); + if (options.NeedLookup() && options.GetMergeEngine() != MergeEngine::FIRST_ROW) { + // don't wrap first row, it is already OK + merge_function = std::make_unique(std::move(merge_function)); + } + std::shared_ptr> merge_function_wrapper = + std::make_shared(std::move(merge_function)); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr sequence_fields_comparator, + PrimaryKeyTableUtils::CreateSequenceFieldsComparator(schema->Fields(), options)); + + std::shared_ptr dv_maintainer_factory; + PAIMON_ASSIGN_OR_RAISE(PrimaryKeyIndexDefinitions primary_key_index_definitions, + PrimaryKeyIndexDefinitions::Create(*schema)); + bool has_btree_index = false; + for (const PrimaryKeyIndexDefinition& definition : + primary_key_index_definitions.Definitions()) { + if (definition.GetFamily() == PrimaryKeyIndexDefinition::Family::BTREE) { + has_btree_index = true; + break; + } + } + std::optional latest_snapshot; + bool has_existing_index_manifest = false; + if (!has_btree_index) { + PAIMON_ASSIGN_OR_RAISE( + bool has_historical_btree_index, + HasHistoricalPrimaryKeyBTreeDefinition(GetSchemaManager(), schema->Id())); + if (has_historical_btree_index) { + PAIMON_ASSIGN_OR_RAISE(latest_snapshot, snapshot_manager->LatestSnapshot()); + has_existing_index_manifest = + latest_snapshot.has_value() && latest_snapshot->IndexManifest().has_value(); + } + } + std::shared_ptr index_file_handler; + if (options.DeletionVectorsEnabled() || has_btree_index || has_existing_index_manifest) { + PAIMON_ASSIGN_OR_RAISE(index_file_handler, store_->NewIndexFileHandler( + options, path_factory, ctx.GetMemoryPool())); + } + bool has_restored_btree_index = false; + if (!has_btree_index && has_existing_index_manifest) { + PAIMON_ASSIGN_OR_RAISE( + std::vector restored_btree_entries, + index_file_handler->Scan( + latest_snapshot.value(), [](const IndexManifestEntry& entry) -> bool { + return entry.index_file->IndexType() == BtreeDefs::kIdentifier && + IndexFileHandler::IsPrimaryKeySourceIndex(*entry.index_file); + })); + has_restored_btree_index = !restored_btree_entries.empty(); + } + if (options.DeletionVectorsEnabled()) { + dv_maintainer_factory = std::make_shared(index_file_handler); + } + std::shared_ptr + primary_key_index_maintainer_factory; + if (has_btree_index || has_restored_btree_index) { + primary_key_index_maintainer_factory = + std::make_shared( + root_path_, CurrentBranch(), schema, primary_key_index_definitions.Definitions(), + path_factory, index_file_handler, options, io_manager, ctx.EnableMultiThreadSpill(), + ctx.GetExecutor(), ctx.GetMemoryPool()); + } + + std::shared_ptr realtime_schema_layout; + if (ctx.GetRealtimeContext()) { + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr layout, + RealtimeSchemaLayout::Create(RealtimeStoreMode::PRIMARY_KEY, arrow_schema)); + realtime_schema_layout = std::move(layout); + } + + return std::make_unique( + path_factory, snapshot_manager, GetSchemaManager(), ctx.GetCommitUser(), root_path_, schema, + arrow_schema, realtime_schema_layout, partition_schema, dv_maintainer_factory, + primary_key_index_maintainer_factory, io_manager, key_comparator, + sequence_fields_comparator, merge_function_wrapper, options, ignore_previous_files, + ctx.IsStreamingMode(), ctx.IgnoreNumBucketCheck(), ctx.EnableMultiThreadSpill(), + ctx.GetRealtimeContext(), ctx.GetExecutor(), ctx.GetMemoryPool()); +} + +Result> PrimaryKeyFileStoreTable::NewFileStoreScan( + const std::shared_ptr& manifest_list, + const std::shared_ptr& manifest_file, + const std::shared_ptr& arrow_schema, + const std::shared_ptr& table_schema, const CoreOptions& core_options, + const std::shared_ptr& executor, const std::shared_ptr& memory_pool, + const ScanContext* context) const { + // TODO(liancheng.lsz): support fallback branch in scan + std::unique_ptr scan; + PAIMON_ASSIGN_OR_RAISE( + scan, KeyValueFileStoreScan::Create(GetSnapshotManager(), GetSchemaManager(), manifest_list, + manifest_file, table_schema, arrow_schema, + context->GetScanFilters(), core_options, executor, + memory_pool)); + return WithTablePath(std::move(scan), root_path_); +} + +Result> PrimaryKeyFileStoreTable::NewSplitGenerator( + const std::shared_ptr& table_schema, const CoreOptions& core_options) const { + auto source_split_target_size = core_options.GetSourceSplitTargetSize(); + auto source_split_open_file_cost = core_options.GetSourceSplitOpenFileCost(); + // TODO(liancheng.lsz): support evolution + PAIMON_ASSIGN_OR_RAISE(std::vector trimmed_primary_keys, + table_schema->TrimmedPrimaryKeys()); + PAIMON_ASSIGN_OR_RAISE(std::vector trimmed_pk_fields, + table_schema->GetFields(trimmed_primary_keys)); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr key_comparator, + FieldsComparator::Create(trimmed_pk_fields, /*is_ascending_order=*/true)); + return std::make_unique( + source_split_target_size, source_split_open_file_cost, + core_options.DeletionVectorsEnabled(), core_options.GetMergeEngine(), key_comparator); +} + +Result> PrimaryKeyFileStoreTable::NewTableRead( + const std::shared_ptr& path_factory, + const std::shared_ptr& internal_context, + const std::shared_ptr& memory_pool, + const std::shared_ptr& executor) const { + return KeyValueTableRead::Create(path_factory, internal_context, memory_pool, executor); +} + +CommitScanner::ScanSupplier PrimaryKeyFileStoreTable::NewCommitScanSupplier( + const std::shared_ptr& snapshot_manager, + const std::shared_ptr& manifest_list, + const std::shared_ptr& manifest_file, + const std::shared_ptr& table_schema, + const std::shared_ptr& arrow_schema, const CoreOptions& options, + const std::shared_ptr& executor, const std::shared_ptr& pool) const { + // The schema manager the table carries, which the scans this supplier hands out read the + // schema through. + const std::shared_ptr& schema_manager = GetSchemaManager(); + return [snapshot_manager, schema_manager, manifest_list, manifest_file, table_schema, + arrow_schema, options, executor, pool](const std::shared_ptr& scan_filter) + -> Result> { + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr scan, + KeyValueFileStoreScan::Create(snapshot_manager, schema_manager, manifest_list, + manifest_file, table_schema, arrow_schema, scan_filter, + options, executor, pool)); + return std::unique_ptr(std::move(scan)); + }; +} + +} // namespace paimon diff --git a/src/paimon/core/table/primary_key_file_store_table.h b/src/paimon/core/table/primary_key_file_store_table.h new file mode 100644 index 000000000..9c658758a --- /dev/null +++ b/src/paimon/core/table/primary_key_file_store_table.h @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include + +#include "paimon/catalog/identifier.h" +#include "paimon/core/table/file_store_table.h" +#include "paimon/core/utils/branch_manager.h" +#include "paimon/result.h" +#include "paimon/type_fwd.h" + +namespace paimon { + +class CoreOptions; +class FileStoreScan; +class IOManager; +class SplitGenerator; +class TableSchema; + +/// The managed table a schema with primary keys describes, Java Paimon's +/// `PrimaryKeyFileStoreTable`. Its write merges by key, and its scan, read and commit read +/// through the merge. The operator flows are the ones `FileStoreTable` carries; the pieces +/// those flows hand down are built here. +class PrimaryKeyFileStoreTable : public FileStoreTable { + public: + /// The constructor `FileStoreTable::Create` builds a primary key table through; the + /// parameters are the ones it takes, and the base class carries them. + PrimaryKeyFileStoreTable(const std::shared_ptr& file_system, + const std::string& root_path, const std::shared_ptr& schema, + const Identifier& identifier, const std::string& uuid = "", + const std::string& branch = BranchManager::DEFAULT_MAIN_BRANCH, + const std::shared_ptr& schema_manager = nullptr, + const std::shared_ptr& snapshot_manager = nullptr); + + private: + Result> CreateWrite( + const WriteContext& ctx, const std::shared_ptr& schema, + const std::shared_ptr& arrow_schema, + const std::shared_ptr& partition_schema, const CoreOptions& options, + const std::shared_ptr& path_factory, + const std::shared_ptr& snapshot_manager, + const std::shared_ptr& io_manager, bool ignore_previous_files) const override; + + Result> NewFileStoreScan( + const std::shared_ptr& manifest_list, + const std::shared_ptr& manifest_file, + const std::shared_ptr& arrow_schema, + const std::shared_ptr& table_schema, const CoreOptions& core_options, + const std::shared_ptr& executor, const std::shared_ptr& memory_pool, + const ScanContext* context) const override; + + Result> NewSplitGenerator( + const std::shared_ptr& table_schema, + const CoreOptions& core_options) const override; + + Result> NewTableRead( + const std::shared_ptr& path_factory, + const std::shared_ptr& internal_context, + const std::shared_ptr& memory_pool, + const std::shared_ptr& executor) const override; + + CommitScanner::ScanSupplier NewCommitScanSupplier( + const std::shared_ptr& snapshot_manager, + const std::shared_ptr& manifest_list, + const std::shared_ptr& manifest_file, + const std::shared_ptr& table_schema, + const std::shared_ptr& arrow_schema, const CoreOptions& options, + const std::shared_ptr& executor, + const std::shared_ptr& pool) const override; +}; + +} // namespace paimon diff --git a/src/paimon/core/table/table_test.cpp b/src/paimon/core/table/table_test.cpp index 01a7ce27e..fe99efe63 100644 --- a/src/paimon/core/table/table_test.cpp +++ b/src/paimon/core/table/table_test.cpp @@ -66,7 +66,7 @@ Result> CreateManagedTable( PAIMON_ASSIGN_OR_RAISE(std::unique_ptr created, schema_manager.CreateTable(schema, /*partition_keys=*/{}, /*primary_keys=*/{}, options)); - return std::make_shared( + return FileStoreTable::Create( file_system, path, checked_pointer_cast(std::shared_ptr(std::move(created))), identifier, /*uuid=*/"", branch); @@ -346,11 +346,13 @@ TEST(TableTest, TestMainBranchAliasResolvesToTheMainBranch) { {Identifier("db", "tbl"), Identifier("db", "tbl$branch_main"), Identifier("db", "tbl$branch_MAIN")}) { SCOPED_TRACE(main_identifier.ToString()); - // A catalog that loaded the schema itself calls the constructor directly, which a + // A catalog that loaded the schema itself builds the table through the factory, which a // rest catalog is shaped like. The alias resolves to the main branch as well. ASSERT_OK_AND_ASSIGN(std::string branch, main_identifier.GetBranchNameOrDefault()); - FileStoreTable table(fs, dir->Str(), schema, main_identifier, /*uuid=*/"", branch); - EXPECT_EQ("main", table.CurrentBranch()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr table, + FileStoreTable::Create(fs, dir->Str(), schema, main_identifier, + /*uuid=*/"", branch)); + EXPECT_EQ("main", table->CurrentBranch()); } } diff --git a/src/paimon/rest/rest_catalog.cpp b/src/paimon/rest/rest_catalog.cpp index 32ebefc1e..b7dcfe04f 100644 --- a/src/paimon/rest/rest_catalog.cpp +++ b/src/paimon/rest/rest_catalog.cpp @@ -471,7 +471,7 @@ Result> RestCatalog::GetTable(const Identifier& identifie // The branch the identifier carries, which the schema and path above were read on, so the // table and its managers serve it. PAIMON_ASSIGN_OR_RAISE(std::optional branch, identifier.GetBranchName()); - return std::make_shared( + return FileStoreTable::Create( table_file_system, table_path, schema, identifier, uuid, NormalizeBranch(std::move(branch)).value_or(Identifier::kDefaultMainBranch)); } diff --git a/src/paimon/testing/mock/mock_catalog.h b/src/paimon/testing/mock/mock_catalog.h index c4a200239..57a6e8d98 100644 --- a/src/paimon/testing/mock/mock_catalog.h +++ b/src/paimon/testing/mock/mock_catalog.h @@ -99,7 +99,7 @@ class MockVersionManagedCatalog : public Catalog, public VersionManagedCatalog { // Serves the per-table file system when one was set, matching `GetTableFileSystem` // without recording the request, so a test can still tell the two apart. No location: // this mock's `GetTableLocation` is the unsupported stub. - return std::make_shared( + return FileStoreTable::Create( table_file_system_ != nullptr ? table_file_system_ : file_system_, /*root_path=*/"", table_schema_, identifier, table_uuid_); } From 09cb5875a46e9a7d6b2e72a46ab131c5a0a1e78d Mon Sep 17 00:00:00 2001 From: "yonghao.fyh" Date: Fri, 23 Oct 2026 13:39:56 +0800 Subject: [PATCH 06/17] tmp --- include/paimon/commit_context.h | 11 +- include/paimon/read_context.h | 11 +- include/paimon/scan_context.h | 12 +- include/paimon/write_context.h | 11 +- .../core/operation/internal_read_context.cpp | 24 +- .../core/operation/internal_read_context.h | 22 ++ .../table/append_only_file_store_table.cpp | 23 +- .../core/table/append_only_file_store_table.h | 22 +- src/paimon/core/table/file_store.cpp | 63 +++- src/paimon/core/table/file_store.h | 84 ++++- src/paimon/core/table/file_store_table.cpp | 312 +++++++++--------- src/paimon/core/table/file_store_table.h | 33 +- .../table/primary_key_file_store_table.cpp | 24 +- .../core/table/primary_key_file_store_table.h | 22 +- .../core/table/source/table_read_test.cpp | 17 +- .../table/system/audit_log_system_table.cpp | 14 +- .../system/read_optimized_system_table.cpp | 14 +- src/paimon/core/table/table_test.cpp | 145 +++++++- src/paimon/rest/rest_catalog.cpp | 18 + test/inte/scan_and_read_inte_test.cpp | 21 +- 20 files changed, 612 insertions(+), 291 deletions(-) diff --git a/include/paimon/commit_context.h b/include/paimon/commit_context.h index 1ec784f76..a70cacb7a 100644 --- a/include/paimon/commit_context.h +++ b/include/paimon/commit_context.h @@ -156,15 +156,18 @@ class PAIMON_EXPORT CommitContextBuilder { ~CommitContextBuilder(); - /// Set a configuration options map to set some option entries which are not defined in the - /// table schema or whose values you want to overwrite. + /// Set a configuration options map. The options take part in loading the table a commit + /// serves: the load merges them over the ones the table's schema carries and the table it + /// hands back serves the merged ones. A caller holding a table already loaded - a load + /// that is over - changes it through `Table::Copy()`; an option contradicting a table held + /// that way is refused when a commit is asked of it. A format table takes the options as + /// its own. /// @note The options map will clear the options added by `AddOption()` before. /// @param options The configuration options map. /// @return Reference to this builder for method chaining. CommitContextBuilder& SetOptions(const std::map& options); - /// Add a single configuration option which is not defined in the table schema or whose value - /// you want to overwrite. + /// Add a single configuration option, served the way `SetOptions()` describes. /// /// If you want to add multiple options, call `AddOption()` multiple times or use `SetOptions()` /// instead. diff --git a/include/paimon/read_context.h b/include/paimon/read_context.h index b9c534065..8cf565996 100644 --- a/include/paimon/read_context.h +++ b/include/paimon/read_context.h @@ -324,15 +324,18 @@ class PAIMON_EXPORT ReadContextBuilder { /// When set, read_field_ids and read_field_names are ignored. ReadContextBuilder& SetReadSchema(std::unique_ptr read_schema); - /// Set a configuration options map to set some option entries which are not defined in the - /// table schema or whose values you want to overwrite. + /// Set a configuration options map. The options take part in loading the table a read + /// serves: the load merges them over the ones the table's schema carries and the table it + /// hands back serves the merged ones. A caller holding a table already loaded - a load + /// that is over - changes it through `Table::Copy()`; an option contradicting a table held + /// that way is refused when a read is asked of it. A format table takes the options as its + /// own. /// @note The options map will clear the options added by `AddOption()` before. /// @param options The configuration options map. /// @return Reference to this builder for method chaining. ReadContextBuilder& SetOptions(const std::map& options); - /// Add a single configuration option which is not defined in the table schema or whose value - /// you want to overwrite. + /// Add a single configuration option, served the way `SetOptions()` describes. /// /// If you want to add multiple options, call `AddOption()` multiple times or use `SetOptions()` /// instead. diff --git a/include/paimon/scan_context.h b/include/paimon/scan_context.h index 5be7c9c4e..fe57afb58 100644 --- a/include/paimon/scan_context.h +++ b/include/paimon/scan_context.h @@ -208,11 +208,15 @@ class PAIMON_EXPORT ScanContextBuilder { ScanContextBuilder& WithRealtimeContext( const std::shared_ptr& realtime_context); - /// The options added or set in `ScanContextBuilder` have high priority and will be merged with - /// the options in table schema. + /// Add a single configuration option. See `SetOptions()` for how the options a context + /// carries are served. ScanContextBuilder& AddOption(const std::string& key, const std::string& value); - /// Set a configuration options map to set some option entries which are not defined in the - /// table schema or whose values you want to overwrite. + /// Set a configuration options map. The options take part in loading the table a scan + /// serves: the load merges them over the ones the table's schema carries and the table it + /// hands back serves the merged ones. A caller holding a table already loaded - a load + /// that is over - changes it through `Table::Copy()`; an option contradicting a table held + /// that way is refused when a scan is asked of it. A format table takes the options as its + /// own. /// @note The options map will clear the options added by `AddOption()` before. /// @param options The configuration options map. /// @return Reference to this builder for method chaining. diff --git a/include/paimon/write_context.h b/include/paimon/write_context.h index 58c0b3f97..23195793f 100644 --- a/include/paimon/write_context.h +++ b/include/paimon/write_context.h @@ -183,15 +183,18 @@ class PAIMON_EXPORT WriteContextBuilder { ~WriteContextBuilder(); - /// Set a configuration options map to set some option entries which are not defined in the - /// table schema or whose values you want to overwrite. + /// Set a configuration options map. The options take part in loading the table a write + /// serves: the load merges them over the ones the table's schema carries and the table it + /// hands back serves the merged ones. A caller holding a table already loaded - a load + /// that is over - changes it through `Table::Copy()`; an option contradicting a table held + /// that way is refused when a write is asked of it. A format table takes the options as its + /// own. /// @note The options map will clear the options added by `AddOption()` before. /// @param options The configuration options map. /// @return Reference to this builder for method chaining. WriteContextBuilder& SetOptions(const std::map& options); - /// Add a single configuration option which is not defined in the table schema or whose value - /// you want to overwrite. + /// Add a single configuration option, served the way `SetOptions()` describes. /// /// If you want to add multiple options, call `AddOption()` multiple times or use `SetOptions()` /// instead. diff --git a/src/paimon/core/operation/internal_read_context.cpp b/src/paimon/core/operation/internal_read_context.cpp index 6c0c43341..778976a46 100644 --- a/src/paimon/core/operation/internal_read_context.cpp +++ b/src/paimon/core/operation/internal_read_context.cpp @@ -273,18 +273,32 @@ std::optional InternalReadContext::TryResolveSpecialFieldByName( return std::nullopt; } +Result> InternalReadContext::Create( + const std::shared_ptr& context, const std::shared_ptr& table_schema, + CoreOptions core_options, const std::string* path) { + core_options.WithCache(context->GetCache()); + return CreateFromCoreOptions(context, table_schema, std::move(core_options), path); +} + Result> InternalReadContext::Create( const std::shared_ptr& context, const std::shared_ptr& table_schema, const std::map& options, const std::shared_ptr& file_system, const std::string* path) { - // The file system a table reads through, or null for the read context's own, which callers - // without a table hand over. + // The file system the caller named, or null for the read context's own, which callers + // without a table hand over. The scheme map the context carries does not take part: a file + // system handed over wins and one missing falls to the resolving one, the same way a scan + // resolves its own. PAIMON_ASSIGN_OR_RAISE( CoreOptions core_options, - CoreOptions::FromMap( - options, file_system != nullptr ? file_system : context->GetSpecificFileSystem(), - context->GetFileSystemSchemeToIdentifierMap())); + CoreOptions::FromMap(options, file_system != nullptr ? file_system + : context->GetSpecificFileSystem())); core_options.WithCache(context->GetCache()); + return CreateFromCoreOptions(context, table_schema, std::move(core_options), path); +} + +Result> InternalReadContext::CreateFromCoreOptions( + const std::shared_ptr& context, const std::shared_ptr& table_schema, + CoreOptions core_options, const std::string* path) { if (context->HasVectorOrFullTextSearch()) { if (!core_options.FileIndexReadEnabled()) { return Status::Invalid( diff --git a/src/paimon/core/operation/internal_read_context.h b/src/paimon/core/operation/internal_read_context.h index b1619c5bc..b7218a686 100644 --- a/src/paimon/core/operation/internal_read_context.h +++ b/src/paimon/core/operation/internal_read_context.h @@ -58,6 +58,21 @@ class InternalReadContext { std::string field_name; }; + /// Builds the context of a read a table serves: the options are the table's own, already + /// parsed - the store's `options()` - and the cache the context carries is armed on them + /// here. `path` names the root path the table was opened with, which the read reads from in + /// place of the one named by the context. + static Result> Create( + const std::shared_ptr& read_context, + const std::shared_ptr& table_schema, CoreOptions core_options, + const std::string* path = nullptr); + + /// Builds the context of a read whose options are composed on the spot rather than the + /// table's own - the fallback branch of a read, whose schema belongs to another branch, and + /// the reads a write or a lookup serve inside themselves. The options are parsed from + /// `options` here: the read's own file system is `file_system`, or the context's when null - + /// the scheme map the context carries does not take part. A cache the context carries is + /// armed on the options this parses. static Result> Create( const std::shared_ptr& read_context, const std::shared_ptr& table_schema, @@ -171,6 +186,13 @@ class InternalReadContext { const std::shared_ptr& arrow_pool, std::optional path = std::nullopt); + /// The tail both `Create` entries share: the options arrive parsed and cached, and the + /// validations and the read schema below are the same whichever way they were resolved. + static Result> CreateFromCoreOptions( + const std::shared_ptr& read_context, + const std::shared_ptr& table_schema, CoreOptions core_options, + const std::string* path); + static std::optional TryResolveSpecialFieldById(int32_t field_id, const CoreOptions& core_options, bool has_vector_or_full_text_search); diff --git a/src/paimon/core/table/append_only_file_store_table.cpp b/src/paimon/core/table/append_only_file_store_table.cpp index 0846aef90..cead79566 100644 --- a/src/paimon/core/table/append_only_file_store_table.cpp +++ b/src/paimon/core/table/append_only_file_store_table.cpp @@ -48,22 +48,19 @@ namespace paimon { -AppendOnlyFileStoreTable::AppendOnlyFileStoreTable( - const std::shared_ptr& file_system, const std::string& root_path, - const std::shared_ptr& schema, const Identifier& identifier, const std::string& uuid, - const std::string& branch, const std::shared_ptr& schema_manager, - const std::shared_ptr& snapshot_manager) - : FileStoreTable(file_system, root_path, schema, identifier, uuid, branch, schema_manager, - snapshot_manager) {} +AppendOnlyFileStoreTable::AppendOnlyFileStoreTable(const std::shared_ptr& store, + const Identifier& identifier, + const std::string& uuid) + : FileStoreTable(store, identifier, uuid) {} Result> AppendOnlyFileStoreTable::CreateWrite( - const WriteContext& ctx, const std::shared_ptr& schema, - const std::shared_ptr& arrow_schema, + const WriteContext& ctx, const std::shared_ptr& arrow_schema, const std::shared_ptr& partition_schema, const CoreOptions& options, const std::shared_ptr& path_factory, const std::shared_ptr& snapshot_manager, const std::shared_ptr& io_manager, bool ignore_previous_files) const { // append table + const std::shared_ptr& schema = store_->schema(); const bool realtime_deduplicate = !options.GetRealtimeDeduplicateKeyFields().empty(); bool need_dv_maintainer_factory = options.DeletionVectorsEnabled(); if (options.GetBucket() == -1) { @@ -138,11 +135,11 @@ Result> AppendOnlyFileStoreTable::CreateWrite( Result> AppendOnlyFileStoreTable::NewFileStoreScan( const std::shared_ptr& manifest_list, const std::shared_ptr& manifest_file, - const std::shared_ptr& arrow_schema, - const std::shared_ptr& table_schema, const CoreOptions& core_options, + const std::shared_ptr& arrow_schema, const CoreOptions& core_options, const std::shared_ptr& executor, const std::shared_ptr& memory_pool, const ScanContext* context) const { // TODO(liancheng.lsz): support fallback branch in scan + const std::shared_ptr& table_schema = store_->schema(); std::unique_ptr scan; if (core_options.DataEvolutionEnabled()) { PAIMON_ASSIGN_OR_RAISE( @@ -161,7 +158,7 @@ Result> AppendOnlyFileStoreTable::NewFileStoreSca } Result> AppendOnlyFileStoreTable::NewSplitGenerator( - const std::shared_ptr& table_schema, const CoreOptions& core_options) const { + const CoreOptions& core_options) const { auto source_split_target_size = core_options.GetSourceSplitTargetSize(); auto source_split_open_file_cost = core_options.GetSourceSplitOpenFileCost(); if (core_options.DataEvolutionEnabled()) { @@ -188,12 +185,12 @@ CommitScanner::ScanSupplier AppendOnlyFileStoreTable::NewCommitScanSupplier( const std::shared_ptr& snapshot_manager, const std::shared_ptr& manifest_list, const std::shared_ptr& manifest_file, - const std::shared_ptr& table_schema, const std::shared_ptr& arrow_schema, const CoreOptions& options, const std::shared_ptr& executor, const std::shared_ptr& pool) const { // The schema manager the table carries, which the scans this supplier hands out read the // schema through. const std::shared_ptr& schema_manager = GetSchemaManager(); + const std::shared_ptr& table_schema = store_->schema(); return [snapshot_manager, schema_manager, manifest_list, manifest_file, table_schema, arrow_schema, options, executor, pool](const std::shared_ptr& scan_filter) -> Result> { diff --git a/src/paimon/core/table/append_only_file_store_table.h b/src/paimon/core/table/append_only_file_store_table.h index ea5374ba3..499431eb0 100644 --- a/src/paimon/core/table/append_only_file_store_table.h +++ b/src/paimon/core/table/append_only_file_store_table.h @@ -24,13 +24,11 @@ #include "paimon/catalog/identifier.h" #include "paimon/core/table/file_store_table.h" -#include "paimon/core/utils/branch_manager.h" #include "paimon/result.h" #include "paimon/type_fwd.h" namespace paimon { -class CoreOptions; class FileStoreScan; class IOManager; class SplitGenerator; @@ -42,19 +40,14 @@ class TableSchema; /// carries; the pieces those flows hand down are built here. class AppendOnlyFileStoreTable : public FileStoreTable { public: - /// The constructor `FileStoreTable::Create` builds an append-only table through; the - /// parameters are the ones it takes, and the base class carries them. - AppendOnlyFileStoreTable(const std::shared_ptr& file_system, - const std::string& root_path, const std::shared_ptr& schema, - const Identifier& identifier, const std::string& uuid = "", - const std::string& branch = BranchManager::DEFAULT_MAIN_BRANCH, - const std::shared_ptr& schema_manager = nullptr, - const std::shared_ptr& snapshot_manager = nullptr); + /// The constructor `FileStoreTable::Create` builds an append-only table through: the store + /// carries every state, and the base class holds it. + AppendOnlyFileStoreTable(const std::shared_ptr& store, const Identifier& identifier, + const std::string& uuid = ""); private: Result> CreateWrite( - const WriteContext& ctx, const std::shared_ptr& schema, - const std::shared_ptr& arrow_schema, + const WriteContext& ctx, const std::shared_ptr& arrow_schema, const std::shared_ptr& partition_schema, const CoreOptions& options, const std::shared_ptr& path_factory, const std::shared_ptr& snapshot_manager, @@ -63,13 +56,11 @@ class AppendOnlyFileStoreTable : public FileStoreTable { Result> NewFileStoreScan( const std::shared_ptr& manifest_list, const std::shared_ptr& manifest_file, - const std::shared_ptr& arrow_schema, - const std::shared_ptr& table_schema, const CoreOptions& core_options, + const std::shared_ptr& arrow_schema, const CoreOptions& core_options, const std::shared_ptr& executor, const std::shared_ptr& memory_pool, const ScanContext* context) const override; Result> NewSplitGenerator( - const std::shared_ptr& table_schema, const CoreOptions& core_options) const override; Result> NewTableRead( @@ -82,7 +73,6 @@ class AppendOnlyFileStoreTable : public FileStoreTable { const std::shared_ptr& snapshot_manager, const std::shared_ptr& manifest_list, const std::shared_ptr& manifest_file, - const std::shared_ptr& table_schema, const std::shared_ptr& arrow_schema, const CoreOptions& options, const std::shared_ptr& executor, const std::shared_ptr& pool) const override; diff --git a/src/paimon/core/table/file_store.cpp b/src/paimon/core/table/file_store.cpp index 80b6afb05..2c5a19d96 100644 --- a/src/paimon/core/table/file_store.cpp +++ b/src/paimon/core/table/file_store.cpp @@ -19,9 +19,11 @@ #include "paimon/core/table/file_store.h" +#include #include #include #include +#include #include #include "paimon/catalog/catalog.h" @@ -39,30 +41,71 @@ #include "paimon/core/utils/file_store_path_factory.h" #include "paimon/core/utils/index_file_path_factories.h" #include "paimon/core/utils/snapshot_manager.h" +#include "paimon/defs.h" #include "paimon/result.h" #include "paimon/status.h" namespace paimon { +Result> FileStore::Create( + const std::shared_ptr& file_system, const std::string& root_path, + const std::shared_ptr& schema, const std::string& branch, + const std::shared_ptr& schema_manager, + const std::shared_ptr& snapshot_manager) { + auto store = std::shared_ptr( + new FileStore(file_system, root_path, schema, branch, schema_manager, snapshot_manager)); + if (schema != nullptr) { + // The options the schema was loaded with, on the branch this store serves: what reads + // through them - whose cache keys are scoped by the branch - reads the same one the + // managers point at. + std::map options = schema->Options(); + if (BranchManager::IsMainBranch(store->branch_)) { + options.erase(Options::BRANCH); + } else { + options[Options::BRANCH] = store->branch_; + } + PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, + CoreOptions::FromMap(options, file_system, {})); + store->core_options_ = std::move(core_options); + } + // The file system the table is served through: the one the options resolve - a schema + // carrying access configuration names its own, which the file system the caller opened the + // table through wins over - and without a schema, the one handed over, whose absence leaves + // the default local one. + if (store->core_options_.has_value()) { + store->file_system_ = store->core_options_->GetFileSystem(); + } else if (file_system != nullptr) { + store->file_system_ = file_system; + } else { + PAIMON_ASSIGN_OR_RAISE(CoreOptions default_options, CoreOptions::FromMap({})); + store->file_system_ = default_options.GetFileSystem(); + } + // The managers read through the same file system: the metadata belongs to the data, and + // both were written under the options the schema carries. A manager handed over already + // points at `root_path` on `branch`; a missing one is built here, so the operators always + // read through the managers the store carries. + if (store->schema_manager_ == nullptr) { + store->schema_manager_ = + std::make_shared(store->file_system_, root_path, store->branch_); + } + if (store->snapshot_manager_ == nullptr) { + store->snapshot_manager_ = + std::make_shared(store->file_system_, root_path, store->branch_); + } + return store; +} + FileStore::FileStore(const std::shared_ptr& file_system, const std::string& root_path, const std::shared_ptr& schema, const std::string& branch, const std::shared_ptr& schema_manager, const std::shared_ptr& snapshot_manager) : file_system_(file_system), + specified_file_system_(file_system), root_path_(root_path), schema_(schema), branch_(BranchManager::NormalizeBranch(branch)), schema_manager_(schema_manager), - snapshot_manager_(snapshot_manager) { - // A manager handed over points at `root_path` on `branch` already; a missing one is built - // here, so the operators always read through the managers the store carries. - if (schema_manager_ == nullptr) { - schema_manager_ = std::make_shared(file_system_, root_path_, branch_); - } - if (snapshot_manager_ == nullptr) { - snapshot_manager_ = std::make_shared(file_system_, root_path_, branch_); - } -} + snapshot_manager_(snapshot_manager) {} Result> FileStore::NewPathFactory( const std::shared_ptr& arrow_schema, const CoreOptions& options, diff --git a/src/paimon/core/table/file_store.h b/src/paimon/core/table/file_store.h index 83c34b1e7..6c2396bd6 100644 --- a/src/paimon/core/table/file_store.h +++ b/src/paimon/core/table/file_store.h @@ -24,6 +24,7 @@ #include #include "paimon/catalog/identifier.h" +#include "paimon/core/core_options.h" #include "paimon/core/schema/schema_manager.h" #include "paimon/core/utils/file_store_path_factory.h" #include "paimon/core/utils/snapshot_manager.h" @@ -38,38 +39,53 @@ namespace paimon { class BinaryRowPartitionComputer; class Catalog; -class CoreOptions; class IndexFileHandler; class IndexManifestFile; class ManifestFile; class ManifestList; class TableSchema; -/// The storage a managed table is served through: the file system, root path, schema and branch -/// it was loaded with, and the schema and snapshot managers that read its root path on that -/// branch - the state Java Paimon's `AbstractFileStore` carries. `FileStoreTable` hands the -/// state it was loaded with to one of these and asks it back for the pieces every operator +/// The storage a managed table is served through: the file system, root path, schema, branch, +/// options and the schema and snapshot managers it was loaded with - the state Java Paimon's +/// `AbstractFileStore` carries, parsed once here rather than per operator. `FileStoreTable` hands +/// the state it was loaded with to one of these and asks it back for the pieces every operator /// opens with. /// +/// The options are the ones the table was loaded with on this branch - the schema's own, or +/// the ones a load merged over them - the base every operator serves: `NewXxx` takes no +/// options of its own beyond them. A cache is not part of them - `WithCache()` arms a copy an +/// operator makes - so the factories below take the options of the call, not the store's own. +/// /// The factory methods build a fresh instance on every call, never a cached one: a path factory /// names each file it builds, and a snapshot manager a write or a commit arms with a catalog /// loader is not one a scan should be served through afterwards. class FileStore { public: - /// @param file_system File system the table's data is accessed through. + /// @param file_system The file system the caller opened the table through, which wins over + /// the access configuration the schema's options may carry when it is not null. A + /// null one hands the resolution to that configuration: the table is then served by + /// the file system its options name - the way a load that only had a path and a + /// schema to read names it - and by the default local one when they name none. /// @param root_path Root path of the table in the file system. /// @param schema The schema the table was loaded with, whose partition keys name the - /// partitions the paths below are built for. + /// partitions the paths below are built for and whose options `options()` parses. It + /// is null when the caller had no schema; the operators then report that none was + /// found. /// @param branch The branch the store serves, normalized here. /// @param schema_manager A schema manager to carry instead of building one - the one a /// context dispatch read the schema through, whose cache the operators then serve. /// @param snapshot_manager A snapshot manager to carry instead of building one. /// A null manager is built over `root_path` on `branch`, so the operators always read /// through the managers the store carries. - FileStore(const std::shared_ptr& file_system, const std::string& root_path, - const std::shared_ptr& schema, const std::string& branch, - const std::shared_ptr& schema_manager = nullptr, - const std::shared_ptr& snapshot_manager = nullptr); + /// + /// Building the store parses the schema's options, so it fails when they do not parse: the + /// error is the caller's to see, which is why a store is built through this factory rather + /// than the constructor. + static Result> Create( + const std::shared_ptr& file_system, const std::string& root_path, + const std::shared_ptr& schema, const std::string& branch, + const std::shared_ptr& schema_manager = nullptr, + const std::shared_ptr& snapshot_manager = nullptr); /// The branch this store serves, as resolved when it was built. const std::string& branch() const { @@ -88,6 +104,40 @@ class FileStore { return snapshot_manager_; } + /// The schema the table was loaded with, which every operator serves rather than reading the + /// schema again. It is null when the caller had no schema. + const std::shared_ptr& schema() const { + return schema_; + } + + /// The file system the table's data is accessed through, as the table was opened with it: + /// the one the options resolve - a schema carrying access configuration names its own, + /// which the file system the caller opened the table through wins over - rather than the + /// one that call handed over, which `specified_file_system()` hands back untouched. + const std::shared_ptr& file_system() const { + return file_system_; + } + + /// The file system the caller opened the table through, as it was handed to `Create`: null + /// when the call had none, so a copy of the table resolves from its own options again and + /// the runtime ones it merges in - a file-system option among them - can speak. + const std::shared_ptr& specified_file_system() const { + return specified_file_system_; + } + + /// The root path of the table in its file system, as the table was opened with it. + const std::string& root_path() const { + return root_path_; + } + + /// The options the schema was loaded with on `branch()`, which every operator serves. They + /// exist only over a schema: an operator reads them after reporting that no schema was + /// found when there is none. + const CoreOptions& options() const { + assert(core_options_.has_value()); + return *core_options_; + } + /// The path factory an operator builds its metadata paths through: over this store's root /// path and the partition keys of its schema - which no schema change can alter, so a read /// serving a fallback branch's schema still names the same partitions. The file format @@ -146,12 +196,24 @@ class FileStore { const std::shared_ptr& memory_pool) const; private: + FileStore(const std::shared_ptr& file_system, const std::string& root_path, + const std::shared_ptr& schema, const std::string& branch, + const std::shared_ptr& schema_manager, + const std::shared_ptr& snapshot_manager); + std::shared_ptr file_system_; + /// The file system the caller opened the table through, as handed to `Create` - null when + /// the call had none - kept for a copy to hand over again: what `file_system_` holds is + /// resolved, and would pin the copy to this table's resolution. + std::shared_ptr specified_file_system_; std::string root_path_; std::shared_ptr schema_; std::string branch_; std::shared_ptr schema_manager_; std::shared_ptr snapshot_manager_; + /// The options of `schema_` on `branch_`, or unset when there is no schema. `Create` parses + /// them: the constructor cannot report a parse failure. + std::optional core_options_; }; } // namespace paimon diff --git a/src/paimon/core/table/file_store_table.cpp b/src/paimon/core/table/file_store_table.cpp index 502556990..2e5f77af6 100644 --- a/src/paimon/core/table/file_store_table.cpp +++ b/src/paimon/core/table/file_store_table.cpp @@ -141,13 +141,45 @@ Result> TryLoadFormatTable( /*location_carries_paimon_metadata=*/true, options); } +/// The options a context carries, merged into the table the load hands back: a table is the +/// product of the load, so the options speak there - the way `Copy` merges them, over the ones +/// the schema carries - and `NewXxx` on the product serves them without merging again. A +/// caller holding a table already - a load that is over - changes it through `Copy` itself, +/// and an option that contradicts the table that way is refused. +Result> WithContextOptions( + const std::shared_ptr
& table, const std::shared_ptr& schema, + const std::map& options) { + if (table == nullptr || schema == nullptr || options.empty()) { + return table; + } + const std::map& schema_options = schema->Options(); + for (const auto& [key, value] : options) { + auto schema_option = schema_options.find(key); + // A key the schema does not carry, or carries differently: the merge has something to + // do. Every key agreeing with the schema leaves the table the load built as it is. + if (schema_option == schema_options.end() || schema_option->second != value) { + return table->Copy(options); + } + } + return table; +} + /// The tail every `TableOf` ends with: the schema one of them read says what the path names. A /// format table is handed back as the `FormatTable` it is, everything else as the /// `FileStoreTable` a managed table is, carrying the schema for its `NewXxx` to build from. A /// null schema is handed over as it is: what to say about a table that is not there is the /// managed path's to decide, and it says it in more detail. +/// +/// @param file_system The file system the schema above was read through, which a format table +/// is served by - it has no schema options of its own to name another one. +/// @param explicit_file_system The file system the context supplied, which a managed table is +/// opened through when there is one. A null one leaves the table to resolve from the +/// options its schema carries: access configuration the schema was stored with then +/// names the files it serves, rather than being pinned to the one this load happened to +/// read the schema through. Result> TableOfLoadedSchema( - const std::shared_ptr& file_system, const std::string& path, + const std::shared_ptr& file_system, + const std::shared_ptr& explicit_file_system, const std::string& path, const std::string& branch, const std::map& options, const std::shared_ptr& schema_manager, const std::shared_ptr& latest_schema, const std::optional& identifier, @@ -161,8 +193,10 @@ Result> TableOfLoadedSchema( // it sits in - the same convention `TryLoadFormatTable` follows above. Identifier table_identifier = identifier.value_or(Identifier(Identifier::kUnknownDatabase, PathUtil::GetName(path))); - return FileStoreTable::Create(file_system, path, latest_schema, table_identifier, uuid, branch, - schema_manager); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table, + FileStoreTable::Create(explicit_file_system, path, latest_schema, + table_identifier, uuid, branch, schema_manager)); + return WithContextOptions(table, latest_schema, options); } /// The managed table a context names behind a path: the schema is read through a manager of the @@ -172,12 +206,14 @@ Result> TableOfLoadedSchema( /// @param schema_json The schema of the table, when the caller already read it, or null when it /// has to be read here. It is honoured on the main branch only: a branch publishes its own /// under `branch/branch-`, which is where the read below goes. -Result> ManagedTableOf(const std::shared_ptr& file_system, - const std::string& path, const std::string& branch, - const std::map& options, - const std::optional& schema_json, - const std::optional& identifier, - const std::string& uuid = "") { +/// @param explicit_file_system The file system the context supplied, which the managed table the +/// schema describes is opened through - see `TableOfLoadedSchema`. +Result> ManagedTableOf( + const std::shared_ptr& file_system, + const std::shared_ptr& explicit_file_system, const std::string& path, + const std::string& branch, const std::map& options, + const std::optional& schema_json, const std::optional& identifier, + const std::string& uuid = "") { auto schema_manager = std::make_shared(file_system, path, branch); std::shared_ptr latest_schema; if (branch == BranchManager::DEFAULT_MAIN_BRANCH && schema_json) { @@ -191,8 +227,8 @@ Result> ManagedTableOf(const std::shared_ptr& latest_schema = *latest; } } - return TableOfLoadedSchema(file_system, path, branch, options, schema_manager, latest_schema, - identifier, uuid); + return TableOfLoadedSchema(file_system, explicit_file_system, path, branch, options, + schema_manager, latest_schema, identifier, uuid); } /// The identifier a catalog knows the table by, on `branch`: a non-main branch is folded into @@ -252,8 +288,9 @@ Result> TableOf(const ScanContext& ctx) { // below. One `TableScan` interface serves both, as Java Paimon serves both through one // `ReadBuilder`. const std::string branch = BranchManager::NormalizeBranch(tmp_options.GetBranch()); - return ManagedTableOf(tmp_options.GetFileSystem(), ctx.GetPath(), branch, ctx.GetOptions(), - ctx.GetSpecificTableSchema(), ctx.GetIdentifier()); + return ManagedTableOf(tmp_options.GetFileSystem(), ctx.GetSpecificFileSystem(), ctx.GetPath(), + branch, ctx.GetOptions(), ctx.GetSpecificTableSchema(), + ctx.GetIdentifier()); } /// The table a read context names, read from what the context carries: a format table the @@ -288,8 +325,9 @@ Result> TableOf(const ReadContext& ctx) { // which files there are and how a row is put back together is its own. How a file is opened // is not: both paths go through `DataFileReaderFactory`. const std::string branch = ctx.GetBranch(); - return ManagedTableOf(tmp_core_options.GetFileSystem(), ctx.GetPath(), branch, ctx.GetOptions(), - ctx.GetSpecificTableSchema(), ctx.GetIdentifier()); + return ManagedTableOf(tmp_core_options.GetFileSystem(), ctx.GetSpecificFileSystem(), + ctx.GetPath(), branch, ctx.GetOptions(), ctx.GetSpecificTableSchema(), + ctx.GetIdentifier()); } /// The table a write context names, read from what the context carries: a format table the @@ -326,8 +364,8 @@ Result> TableOf(const WriteContext& ctx) { CatalogMainBranchSchemaJson(ctx.GetCatalog(), catalog_identifier, branch, "write")); // A format table writes plain data files into a directory, so it never reaches the manifest // path below. - return ManagedTableOf(tmp_options.GetFileSystem(), ctx.GetRootPath(), branch, ctx.GetOptions(), - catalog_table_schema, ctx.GetIdentifier()); + return ManagedTableOf(tmp_options.GetFileSystem(), specific_fs, ctx.GetRootPath(), branch, + ctx.GetOptions(), catalog_table_schema, ctx.GetIdentifier()); } /// The table a commit context names, read from what the context carries: a format table the @@ -369,15 +407,18 @@ Result> TableOf(const CommitContext& ctx) { // A format table commits by renaming files into place, so it never reaches the snapshot path // below. The managed path here reads the commit's branch, so this reads the same one: the two // must not dispatch on different schemas. - return ManagedTableOf(tmp_options.GetFileSystem(), root_path, branch, ctx.GetOptions(), - catalog_table_schema, ctx.GetIdentifier(), ctx.GetTableId().value_or("")); + return ManagedTableOf(tmp_options.GetFileSystem(), specific_fs, root_path, branch, + ctx.GetOptions(), catalog_table_schema, ctx.GetIdentifier(), + ctx.GetTableId().value_or("")); } /// The options a context carries against the ones the table was loaded with. A table serves the /// options it was loaded with, so an option the context contradicts is refused rather than -/// honoured - a caller that means to change them copies the table. A key the table was not loaded -/// with is no contradiction: it names a runtime setting - a scan mode, a batch size - which is -/// none of the table's business, and is served as given. +/// honoured - a caller that means to change them copies the table. A table a load hands back +/// carries the context's own options merged in, so it agrees with the context by construction; +/// the refusal belongs to a table the caller held already, whose load is over. A key the table +/// was not loaded with is no contradiction: it names a runtime setting - a scan mode, a batch +/// size - which is none of the table's business, and is served as given. Status CheckOptionsAgreeWithTable(const TableSchema& table_schema, const std::map& context_options) { const std::map& table_options = table_schema.Options(); @@ -409,33 +450,6 @@ Status CheckOperatorContext(const ContextPointer& context, const char* kind) { return Status::OK(); } -/// The schema the table was constructed with, which a write or a commit serves as the table was -/// loaded rather than reading the schema again. -Result> LatestTableSchemaOfBranch( - const std::shared_ptr& schema, const std::string& branch) { - std::shared_ptr latest_schema = std::static_pointer_cast(schema); - if (latest_schema == nullptr) { - return Status::Invalid(fmt::format("not found latest schema in branch {}", branch)); - } - return latest_schema; -} - -/// The options a write or a commit serves: the ones the table was loaded with - an option the -/// context contradicts is refused by `CheckOptionsAgreeWithTable` above - the ones the context -/// adds on top, and the table's branch, which the metadata paths below are built for: what reads -/// through these options, whose cache keys are scoped by it, has to read the same one. -Result> MergedOperatorOptions( - const TableSchema& table_schema, const std::map& context_options, - const std::string& branch) { - PAIMON_RETURN_NOT_OK(CheckOptionsAgreeWithTable(table_schema, context_options)); - auto opts = table_schema.Options(); - for (const auto& [key, value] : context_options) { - opts[key] = value; - } - opts[Options::BRANCH] = branch; - return opts; -} - } // namespace Table::Table(const std::shared_ptr& file_system, const std::string& root_path, @@ -447,16 +461,10 @@ Table::Table(const std::shared_ptr& file_system, const std::string& identifier_(identifier), uuid_(uuid) {} -FileStoreTable::FileStoreTable(const std::shared_ptr& file_system, - const std::string& root_path, const std::shared_ptr& schema, - const Identifier& identifier, const std::string& uuid, - const std::string& branch, - const std::shared_ptr& schema_manager, - const std::shared_ptr& snapshot_manager) - : Table(file_system, root_path, schema, identifier, uuid), - store_(std::make_shared(file_system, root_path, - std::static_pointer_cast(schema), branch, - schema_manager, snapshot_manager)) {} +FileStoreTable::FileStoreTable(const std::shared_ptr& store, + const Identifier& identifier, const std::string& uuid) + : Table(store->file_system(), store->root_path(), store->schema(), identifier, uuid), + store_(store) {} Result> FileStoreTable::Create( const std::shared_ptr& file_system, const std::string& root_path, @@ -466,15 +474,25 @@ Result> FileStoreTable::Create( // The kind the schema's primary keys name, which no schema change can add or drop, so the // kind the table is built as is the kind every schema its root path ever held describes. A // caller without a schema gets the append-only kind, whose operators then report that no - // schema was found. - std::shared_ptr table_schema = std::static_pointer_cast(schema); + // schema was found. A schema that is not a table schema - the read-only one a system table + // exposes, say - is none to serve a managed table through, so it is refused rather than cast. + std::shared_ptr table_schema; + if (schema != nullptr) { + table_schema = std::dynamic_pointer_cast(schema); + if (table_schema == nullptr) { + return Status::Invalid( + fmt::format("{} is not a managed table schema", identifier.ToString())); + } + } + // The store parses the schema's options, so a schema whose options do not parse is refused + // here: the table it names cannot be served, not by one operator more than another. + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr store, + FileStore::Create(file_system, root_path, table_schema, branch, + schema_manager, snapshot_manager)); if (table_schema != nullptr && !table_schema->PrimaryKeys().empty()) { - return std::make_shared(file_system, root_path, schema, - identifier, uuid, branch, schema_manager, - snapshot_manager); + return std::make_shared(store, identifier, uuid); } - return std::make_shared( - file_system, root_path, schema, identifier, uuid, branch, schema_manager, snapshot_manager); + return std::make_shared(store, identifier, uuid); } Result> FileStoreTable::Copy( @@ -490,8 +508,11 @@ Result> FileStoreTable::Copy( PAIMON_ASSIGN_OR_RAISE(std::unique_ptr merged_schema, loaded_schema->MergeOptions(overridable)); // The managers are handed over, not rebuilt: the copy serves the same root path and branch, - // so its operators read the schema cache this table already warmed. - return FileStoreTable::Create(file_system_, root_path_, + // so its operators read the schema cache this table already warmed. The file system is the + // one the caller opened this table through, not the one it resolved to: the copy resolves + // from its own merged options again, so a file-system option among the ones it was given + // speaks rather than being pinned to this table's resolution. + return FileStoreTable::Create(store_->specified_file_system(), root_path_, std::shared_ptr(std::move(merged_schema)), identifier_, CatalogUuid().value_or(""), CurrentBranch(), GetSchemaManager(), GetSnapshotManager()); @@ -502,8 +523,8 @@ Result> FileStoreTable::SwitchToBranch(const std::string& if (target_branch == CurrentBranch()) { // The table already serves this branch: a copy of it - schema and managers shared - is // the same table to the caller. - return FileStoreTable::Create(file_system_, root_path_, schema_, identifier_, - CatalogUuid().value_or(""), CurrentBranch(), + return FileStoreTable::Create(store_->specified_file_system(), root_path_, schema_, + identifier_, CatalogUuid().value_or(""), CurrentBranch(), GetSchemaManager(), GetSnapshotManager()); } // The schema of the branch switched to, read from its own directory, which a branch without @@ -522,9 +543,10 @@ Result> FileStoreTable::SwitchToBranch(const std::string& } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr branch_schema, (*latest)->MergeOptions(overridable)); - return FileStoreTable::Create( - file_system_, root_path_, std::shared_ptr(std::move(branch_schema)), - identifier_, CatalogUuid().value_or(""), target_branch, schema_manager); + return FileStoreTable::Create(store_->specified_file_system(), root_path_, + std::shared_ptr(std::move(branch_schema)), + identifier_, CatalogUuid().value_or(""), target_branch, + schema_manager); } Status FileStoreTable::RestoreRealtimeCommittedProgress( @@ -568,19 +590,18 @@ Result> FileStoreTable::NewWrite( MakeBranchCatalogIdentifier(ctx->GetIdentifier(), CurrentBranch())); // The schema the table was constructed with, so a write serves the table as it was loaded // rather than reading the schema again. - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, - LatestTableSchemaOfBranch(schema_, CurrentBranch())); + const std::shared_ptr& schema = store_->schema(); + if (schema == nullptr) { + return Status::Invalid( + fmt::format("not found latest schema in branch {}", CurrentBranch())); + } + // The options the store was built with are the ones this write serves: an option the + // context contradicts is refused rather than honoured, and a key the table was not loaded + // with names a runtime setting the write does not take - `Copy` derives the table that one + // belongs to. + PAIMON_RETURN_NOT_OK(CheckOptionsAgreeWithTable(*schema, ctx->GetOptions())); + CoreOptions options = store_->options(); auto arrow_schema = DataField::ConvertDataFieldsToArrowSchema(schema->Fields()); - // The options the table was loaded with, with a runtime setting the context adds on top: an - // option the context would have changed is refused rather than honoured, and the branch is - // the table's - the metadata paths below are built for it, so what reads through these - // options, whose cache keys are scoped by it, has to read the same one. - std::map opts; - PAIMON_ASSIGN_OR_RAISE(opts, - MergedOperatorOptions(*schema, ctx->GetOptions(), CurrentBranch())); - PAIMON_ASSIGN_OR_RAISE( - CoreOptions options, - CoreOptions::FromMap(opts, file_system_, ctx->GetFileSystemSchemeToIdentifierMap())); PAIMON_RETURN_NOT_OK(SchemaValidation::ValidateRealtime(*schema, options)); PAIMON_RETURN_NOT_OK(SchemaValidation::ValidateVectorStore(*schema, options)); if (options.GetVectorFileFormat() && !ctx->GetWriteSchema().empty()) { @@ -613,9 +634,8 @@ Result> FileStoreTable::NewWrite( RealtimeContextImpl::Cast(ctx->GetRealtimeContext())); PAIMON_RETURN_NOT_OK(realtime_context_impl->CheckUsable()); } - return CreateWrite(*ctx, schema, arrow_schema, partition_schema, options, - file_store_path_factory, snapshot_manager, io_manager, - ignore_previous_files); + return CreateWrite(*ctx, arrow_schema, partition_schema, options, file_store_path_factory, + snapshot_manager, io_manager, ignore_previous_files); } Result> FileStoreWrite::Create(std::unique_ptr ctx) { @@ -676,34 +696,22 @@ Result> FileStoreTable::NewScan( PAIMON_RETURN_NOT_OK(CheckOperatorContext(context, "scan")); // The schema the table was constructed with, so a scan serves the table as it was loaded // rather than reading the schema again. - std::shared_ptr loaded_schema = std::static_pointer_cast(schema_); - if (loaded_schema == nullptr) { + const std::shared_ptr& table_schema = store_->schema(); + if (table_schema == nullptr) { return Status::Invalid("not found latest schema"); } - const std::shared_ptr& table_schema = loaded_schema; PAIMON_RETURN_NOT_OK(CheckOptionsAgreeWithTable(*table_schema, context->GetOptions())); PAIMON_ASSIGN_OR_RAISE(bool read_optimized, OptionsUtils::GetValueFromMap(context->GetOptions(), kReadOptimizedScanOption, false)); - // The options the table was loaded with, with a runtime setting the context adds on top: an - // option the context would have changed is refused above, so the table's own stand as they - // are. The branch is the table's, not the context's: a table is the branch it was loaded - // for, so a branch the context carries is not honoured. - auto options = table_schema->Options(); - for (const auto& [key, value] : context->GetOptions()) { - if (key != Options::BRANCH) { - options[key] = value; - } - } - if (BranchManager::IsMainBranch(CurrentBranch())) { - options.erase(Options::BRANCH); - } else { - options[Options::BRANCH] = CurrentBranch(); - } - // The file system and root path the table was opened with, so a scan serves the table as it - // was loaded rather than resolving them from the context. - PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, - CoreOptions::FromMap(options, file_system_, {})); + // The options the store was built with are the ones this scan serves: an option the context + // contradicts is refused above, and a key the table was not loaded with names a runtime + // setting the scan does not take - `Copy` derives the table that one belongs to. The branch + // in them is the table's, not the context's: a table is the branch it was loaded for, so a + // branch the context carries is not honoured. + CoreOptions core_options = store_->options(); + // The cache is a runtime setting of this scan, not of the table: the copy the scan serves + // carries it. core_options.WithCache(context->GetCache()); PAIMON_RETURN_NOT_OK(SchemaValidation::ValidateRealtime(*table_schema, core_options)); @@ -747,10 +755,10 @@ Result> FileStoreTable::NewScan( // The managers the table carries, whose schema cache the dispatch already warmed. PAIMON_ASSIGN_OR_RAISE( std::shared_ptr file_store_scan, - NewFileStoreScan(manifest_list, manifest_file, arrow_schema, table_schema, core_options, + NewFileStoreScan(manifest_list, manifest_file, arrow_schema, core_options, context->GetExecutor(), context->GetMemoryPool(), context.get())); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr split_generator, - NewSplitGenerator(table_schema, core_options)); + NewSplitGenerator(core_options)); PAIMON_ASSIGN_OR_RAISE( std::unique_ptr index_file_handler, store_->NewIndexFileHandler(core_options, path_factory, context->GetMemoryPool())); @@ -802,28 +810,24 @@ Result> TableScan::Create(std::unique_ptr> CreateInternalReadContext( +/// The read context of a fallback branch, whose schema is not the table's own: it is read +/// through a manager of that branch, and the options the read serves are the ones that schema +/// was loaded with - the same way the table's own read serves the store's - with the branch it +/// was read from in them. A branch the context carries is not honoured here. +/// @param file_system The file system the table was opened with, which the fallback reads +/// through in place of one resolved from the context. +/// @param root_path The root path the table was opened with, which the fallback reads from in +/// place of the one named by the context. +Result> CreateFallbackReadContext( const std::shared_ptr& context, const std::string& branch, - const std::shared_ptr& schema_manager, - const std::shared_ptr& loaded_schema, const std::shared_ptr& file_system, const std::string& root_path) { std::shared_ptr table_schema; const auto& specific_table_schema = context->GetSpecificTableSchema(); - if (loaded_schema != nullptr) { - table_schema = loaded_schema; - } else if (branch == BranchManager::DEFAULT_MAIN_BRANCH && specific_table_schema) { + if (branch == BranchManager::DEFAULT_MAIN_BRANCH && specific_table_schema) { PAIMON_ASSIGN_OR_RAISE(table_schema, TableSchema::CreateFromJson(specific_table_schema.value())); } else { + auto schema_manager = std::make_shared(file_system, root_path, branch); PAIMON_ASSIGN_OR_RAISE(std::optional> latest_schema, schema_manager->Latest()); if (!latest_schema) { @@ -833,16 +837,7 @@ Result> CreateInternalReadContext( table_schema = latest_schema.value(); } assert(table_schema); - - // The options the schema was loaded with, with a runtime setting the context adds on top - - // and the fallback below serves the ones its own schema was loaded with. The branch is the - // one named below, not the context's: a branch the context carries is not honoured here. auto options = table_schema->Options(); - for (const auto& [key, value] : context->GetOptions()) { - if (key != Options::BRANCH) { - options[key] = value; - } - } if (branch != BranchManager::DEFAULT_MAIN_BRANCH) { options[Options::BRANCH] = branch; } @@ -881,18 +876,20 @@ Result> FileStoreTable::NewRead( PAIMON_RETURN_NOT_OK(CheckOperatorContext(read_context, "read")); // The schema the table was constructed with, so a read serves the table as it was loaded // rather than reading the schema again. Its options are the ones the read serves: an option - // the context would have changed is refused rather than honoured. - std::shared_ptr loaded_schema = std::static_pointer_cast(schema_); + // the context would have changed is refused rather than honoured, and a key the table was + // not loaded with names a runtime setting the read does not take - `Copy` derives the table + // that one belongs to. + const std::shared_ptr& loaded_schema = store_->schema(); if (loaded_schema == nullptr) { return Status::Invalid("not found latest schema"); } PAIMON_RETURN_NOT_OK(CheckOptionsAgreeWithTable(*loaded_schema, read_context->GetOptions())); - // The branch the table serves, which the read reads on: a table is the branch it was loaded - // for, so the context's own branch is not consulted. + // The options the store was built with, which the read reads through on the branch the + // table serves: a table is the branch it was loaded for, so the context's own branch is + // not consulted. PAIMON_ASSIGN_OR_RAISE( std::shared_ptr internal_context, - CreateInternalReadContext(read_context, CurrentBranch(), GetSchemaManager(), loaded_schema, - file_system_, root_path_)); + InternalReadContext::Create(read_context, loaded_schema, store_->options(), &root_path_)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_read, CreateTableRead(internal_context, read_context->GetMemoryPool(), @@ -909,10 +906,8 @@ Result> FileStoreTable::NewRead( // own rather than the table's. PAIMON_ASSIGN_OR_RAISE( std::shared_ptr fallback_context, - CreateInternalReadContext( - read_context, /*branch=*/scan_fallback_branch.value(), - std::make_shared(file_system_, root_path_, scan_fallback_branch.value()), - /*loaded_schema=*/nullptr, file_system_, root_path_)); + CreateFallbackReadContext(read_context, /*branch=*/scan_fallback_branch.value(), + file_system_, root_path_)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr fallback_table_read, CreateTableRead(fallback_context, read_context->GetMemoryPool(), @@ -1002,18 +997,19 @@ Result> FileStoreTable::NewCommit( } // The schema the table was constructed with, so a commit serves the table as it was loaded // rather than reading the schema again. - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, - LatestTableSchemaOfBranch(schema_, CurrentBranch())); + const std::shared_ptr& schema = store_->schema(); + if (schema == nullptr) { + return Status::Invalid( + fmt::format("not found latest schema in branch {}", CurrentBranch())); + } std::shared_ptr arrow_schema = DataField::ConvertDataFieldsToArrowSchema(schema->Fields()); - // The options the table was loaded with, with a runtime setting the context adds on top: an - // option the context would have changed is refused rather than honoured, and the branch is - // the table's - the metadata paths below are built for it, so what reads through these - // options, whose cache keys are scoped by it, has to read the same one. - std::map opts; - PAIMON_ASSIGN_OR_RAISE(opts, - MergedOperatorOptions(*schema, ctx->GetOptions(), CurrentBranch())); - PAIMON_ASSIGN_OR_RAISE(CoreOptions options, CoreOptions::FromMap(opts, file_system_)); + // The options the store was built with are the ones this commit serves: an option the + // context contradicts is refused rather than honoured, and a key the table was not loaded + // with names a runtime setting the commit does not take - `Copy` derives the table that one + // belongs to. + PAIMON_RETURN_NOT_OK(CheckOptionsAgreeWithTable(*schema, ctx->GetOptions())); + CoreOptions options = store_->options(); PAIMON_RETURN_NOT_OK(SchemaValidation::ValidateRealtime(*schema, options)); assert(options.GetFileSystem()); assert(options.GetFileFormat()); @@ -1056,8 +1052,8 @@ Result> FileStoreTable::NewCommit( ctx->GetExecutor()); CommitScanner::ScanSupplier scan_supplier = - NewCommitScanSupplier(snapshot_manager, manifest_list, manifest_file, schema, arrow_schema, - options, ctx->GetExecutor(), ctx->GetMemoryPool()); + NewCommitScanSupplier(snapshot_manager, manifest_list, manifest_file, arrow_schema, options, + ctx->GetExecutor(), ctx->GetMemoryPool()); PAIMON_ASSIGN_OR_RAISE( std::shared_ptr snapshot_commit, diff --git a/src/paimon/core/table/file_store_table.h b/src/paimon/core/table/file_store_table.h index 00959d257..8df9c4d86 100644 --- a/src/paimon/core/table/file_store_table.h +++ b/src/paimon/core/table/file_store_table.h @@ -148,13 +148,11 @@ class FileStoreTable : public Table { const std::shared_ptr& executor) const; protected: - /// The base constructor the kinds of managed table build through, `Create` above. The - /// parameters are the ones it takes; see there for what each one is. - FileStoreTable(const std::shared_ptr& file_system, const std::string& root_path, - const std::shared_ptr& schema, const Identifier& identifier, - const std::string& uuid, const std::string& branch, - const std::shared_ptr& schema_manager, - const std::shared_ptr& snapshot_manager); + /// The base constructor the kinds of managed table build through, `Create` above: the store + /// the table was loaded with carries every state the kinds hand down - see `Create` for what + /// each piece of it is. + FileStoreTable(const std::shared_ptr& store, const Identifier& identifier, + const std::string& uuid); /// The committed progress a real-time write restores, from the latest snapshot the snapshot /// manager reads: where the write picks up after, which both kinds of real-time write ask @@ -170,32 +168,32 @@ class FileStoreTable : public Table { /// The write of this kind of table, which `NewWrite` hands the shared flow's pieces to: the /// kind's writer, built over the path factory, snapshot manager and options that flow - /// resolved. The branch the write serves is the table's, `CurrentBranch()`. + /// resolved. The schema is the store's own, the one the table was loaded with, and the + /// branch the write serves is the table's, `CurrentBranch()`. /// @param io_manager The spill manager the context's temp directory names, or null when the /// context named none. /// @param ignore_previous_files Whether the write starts over rather than appending, which /// the flow resolved with this kind's own rules. virtual Result> CreateWrite( - const WriteContext& ctx, const std::shared_ptr& schema, - const std::shared_ptr& arrow_schema, + const WriteContext& ctx, const std::shared_ptr& arrow_schema, const std::shared_ptr& partition_schema, const CoreOptions& options, const std::shared_ptr& path_factory, const std::shared_ptr& snapshot_manager, const std::shared_ptr& io_manager, bool ignore_previous_files) const = 0; /// The scan of this kind of table, which `NewScan` hands the manifest list and file it built - /// to. The snapshot and schema managers are the table's own, which the scan serves. + /// to. The schema is the store's own and the snapshot and schema managers are the table's + /// own, which the scan serves. virtual Result> NewFileStoreScan( const std::shared_ptr& manifest_list, const std::shared_ptr& manifest_file, - const std::shared_ptr& arrow_schema, - const std::shared_ptr& table_schema, const CoreOptions& core_options, + const std::shared_ptr& arrow_schema, const CoreOptions& core_options, const std::shared_ptr& executor, const std::shared_ptr& memory_pool, const ScanContext* context) const = 0; - /// The split generator of this kind of table, Java Paimon's `splitGenerator()`. + /// The split generator of this kind of table, Java Paimon's `splitGenerator()`. The schema + /// it reads its kind's keys from is the store's own. virtual Result> NewSplitGenerator( - const std::shared_ptr& table_schema, const CoreOptions& core_options) const = 0; /// The read of this kind of table, which `NewRead` hands the internal context it read the @@ -207,13 +205,12 @@ class FileStoreTable : public Table { const std::shared_ptr& executor) const = 0; /// The manifest entry scan supplier of this kind of table, which `NewCommit` hands the - /// manifest list and file it built to. The schema manager the scans it hands out read through - /// is the table's own. + /// manifest list and file it built to. The schema is the store's own, and the schema manager + /// the scans it hands out read through is the table's own. virtual CommitScanner::ScanSupplier NewCommitScanSupplier( const std::shared_ptr& snapshot_manager, const std::shared_ptr& manifest_list, const std::shared_ptr& manifest_file, - const std::shared_ptr& table_schema, const std::shared_ptr& arrow_schema, const CoreOptions& options, const std::shared_ptr& executor, const std::shared_ptr& pool) const = 0; diff --git a/src/paimon/core/table/primary_key_file_store_table.cpp b/src/paimon/core/table/primary_key_file_store_table.cpp index 564e015e3..06e27e0d1 100644 --- a/src/paimon/core/table/primary_key_file_store_table.cpp +++ b/src/paimon/core/table/primary_key_file_store_table.cpp @@ -96,22 +96,19 @@ Result HasHistoricalPrimaryKeyBTreeDefinition( } // namespace -PrimaryKeyFileStoreTable::PrimaryKeyFileStoreTable( - const std::shared_ptr& file_system, const std::string& root_path, - const std::shared_ptr& schema, const Identifier& identifier, const std::string& uuid, - const std::string& branch, const std::shared_ptr& schema_manager, - const std::shared_ptr& snapshot_manager) - : FileStoreTable(file_system, root_path, schema, identifier, uuid, branch, schema_manager, - snapshot_manager) {} +PrimaryKeyFileStoreTable::PrimaryKeyFileStoreTable(const std::shared_ptr& store, + const Identifier& identifier, + const std::string& uuid) + : FileStoreTable(store, identifier, uuid) {} Result> PrimaryKeyFileStoreTable::CreateWrite( - const WriteContext& ctx, const std::shared_ptr& schema, - const std::shared_ptr& arrow_schema, + const WriteContext& ctx, const std::shared_ptr& arrow_schema, const std::shared_ptr& partition_schema, const CoreOptions& options, const std::shared_ptr& path_factory, const std::shared_ptr& snapshot_manager, const std::shared_ptr& io_manager, bool ignore_previous_files) const { // pk table + const std::shared_ptr& schema = store_->schema(); if (ctx.GetRealtimeContext()) { PAIMON_RETURN_NOT_OK(PrimaryKeyTableUtils::ValidateRealtimeOptions(options, *schema)); if (ignore_previous_files) { @@ -227,11 +224,11 @@ Result> PrimaryKeyFileStoreTable::CreateWrite( Result> PrimaryKeyFileStoreTable::NewFileStoreScan( const std::shared_ptr& manifest_list, const std::shared_ptr& manifest_file, - const std::shared_ptr& arrow_schema, - const std::shared_ptr& table_schema, const CoreOptions& core_options, + const std::shared_ptr& arrow_schema, const CoreOptions& core_options, const std::shared_ptr& executor, const std::shared_ptr& memory_pool, const ScanContext* context) const { // TODO(liancheng.lsz): support fallback branch in scan + const std::shared_ptr& table_schema = store_->schema(); std::unique_ptr scan; PAIMON_ASSIGN_OR_RAISE( scan, KeyValueFileStoreScan::Create(GetSnapshotManager(), GetSchemaManager(), manifest_list, @@ -242,10 +239,11 @@ Result> PrimaryKeyFileStoreTable::NewFileStoreSca } Result> PrimaryKeyFileStoreTable::NewSplitGenerator( - const std::shared_ptr& table_schema, const CoreOptions& core_options) const { + const CoreOptions& core_options) const { auto source_split_target_size = core_options.GetSourceSplitTargetSize(); auto source_split_open_file_cost = core_options.GetSourceSplitOpenFileCost(); // TODO(liancheng.lsz): support evolution + const std::shared_ptr& table_schema = store_->schema(); PAIMON_ASSIGN_OR_RAISE(std::vector trimmed_primary_keys, table_schema->TrimmedPrimaryKeys()); PAIMON_ASSIGN_OR_RAISE(std::vector trimmed_pk_fields, @@ -270,12 +268,12 @@ CommitScanner::ScanSupplier PrimaryKeyFileStoreTable::NewCommitScanSupplier( const std::shared_ptr& snapshot_manager, const std::shared_ptr& manifest_list, const std::shared_ptr& manifest_file, - const std::shared_ptr& table_schema, const std::shared_ptr& arrow_schema, const CoreOptions& options, const std::shared_ptr& executor, const std::shared_ptr& pool) const { // The schema manager the table carries, which the scans this supplier hands out read the // schema through. const std::shared_ptr& schema_manager = GetSchemaManager(); + const std::shared_ptr& table_schema = store_->schema(); return [snapshot_manager, schema_manager, manifest_list, manifest_file, table_schema, arrow_schema, options, executor, pool](const std::shared_ptr& scan_filter) -> Result> { diff --git a/src/paimon/core/table/primary_key_file_store_table.h b/src/paimon/core/table/primary_key_file_store_table.h index 9c658758a..9b0ed5c9a 100644 --- a/src/paimon/core/table/primary_key_file_store_table.h +++ b/src/paimon/core/table/primary_key_file_store_table.h @@ -24,13 +24,11 @@ #include "paimon/catalog/identifier.h" #include "paimon/core/table/file_store_table.h" -#include "paimon/core/utils/branch_manager.h" #include "paimon/result.h" #include "paimon/type_fwd.h" namespace paimon { -class CoreOptions; class FileStoreScan; class IOManager; class SplitGenerator; @@ -42,19 +40,14 @@ class TableSchema; /// those flows hand down are built here. class PrimaryKeyFileStoreTable : public FileStoreTable { public: - /// The constructor `FileStoreTable::Create` builds a primary key table through; the - /// parameters are the ones it takes, and the base class carries them. - PrimaryKeyFileStoreTable(const std::shared_ptr& file_system, - const std::string& root_path, const std::shared_ptr& schema, - const Identifier& identifier, const std::string& uuid = "", - const std::string& branch = BranchManager::DEFAULT_MAIN_BRANCH, - const std::shared_ptr& schema_manager = nullptr, - const std::shared_ptr& snapshot_manager = nullptr); + /// The constructor `FileStoreTable::Create` builds a primary key table through: the store + /// carries every state, and the base class holds it. + PrimaryKeyFileStoreTable(const std::shared_ptr& store, const Identifier& identifier, + const std::string& uuid = ""); private: Result> CreateWrite( - const WriteContext& ctx, const std::shared_ptr& schema, - const std::shared_ptr& arrow_schema, + const WriteContext& ctx, const std::shared_ptr& arrow_schema, const std::shared_ptr& partition_schema, const CoreOptions& options, const std::shared_ptr& path_factory, const std::shared_ptr& snapshot_manager, @@ -63,13 +56,11 @@ class PrimaryKeyFileStoreTable : public FileStoreTable { Result> NewFileStoreScan( const std::shared_ptr& manifest_list, const std::shared_ptr& manifest_file, - const std::shared_ptr& arrow_schema, - const std::shared_ptr& table_schema, const CoreOptions& core_options, + const std::shared_ptr& arrow_schema, const CoreOptions& core_options, const std::shared_ptr& executor, const std::shared_ptr& memory_pool, const ScanContext* context) const override; Result> NewSplitGenerator( - const std::shared_ptr& table_schema, const CoreOptions& core_options) const override; Result> NewTableRead( @@ -82,7 +73,6 @@ class PrimaryKeyFileStoreTable : public FileStoreTable { const std::shared_ptr& snapshot_manager, const std::shared_ptr& manifest_list, const std::shared_ptr& manifest_file, - const std::shared_ptr& table_schema, const std::shared_ptr& arrow_schema, const CoreOptions& options, const std::shared_ptr& executor, const std::shared_ptr& pool) const override; diff --git a/src/paimon/core/table/source/table_read_test.cpp b/src/paimon/core/table/source/table_read_test.cpp index 3d580d64d..b6aef3992 100644 --- a/src/paimon/core/table/source/table_read_test.cpp +++ b/src/paimon/core/table/source/table_read_test.cpp @@ -251,7 +251,7 @@ TEST(TableReadTest, TestCreateAppendOnlyTableRead) { TEST(TableReadTest, TestMergeOptions) { std::string path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09"; { - // The options the table was loaded with stand as they are, and a runtime setting the + // The options the schema carries stand in the table the load builds, and a setting the // context adds on top - the table was loaded with neither of these keys - is served as // given. ReadContextBuilder context_builder(path); @@ -276,14 +276,21 @@ TEST(TableReadTest, TestMergeOptions) { ASSERT_EQ(expected_options, core_options.ToMap()); } { - // An option that contradicts the table is refused: the read serves the options the table - // was loaded with, and a caller that means to change them copies the table. + // An option the schema also carries speaks at the load: the table the context is + // handed back is built with the context's value over the schema's, and the read serves + // the merged options. A caller holding a table already - a load that is over - is the + // one that gets a refusal instead, which `TableTest` covers. ReadContextBuilder context_builder(path); context_builder.SetReadFieldNames({"f0", "f1", "f2", "f3"}); context_builder.AddOption("bucket", "10"); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(read_context)), - "option 'bucket' is '10' in the context but '2' on the table"); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + auto append_only_table_read = dynamic_cast(table_read.get()); + ASSERT_TRUE(append_only_table_read); + auto* typed_split_read = + dynamic_cast(append_only_table_read->split_reads_[0].get()); + ASSERT_TRUE(typed_split_read); + ASSERT_EQ(10, typed_split_read->options_.GetBucket()); } } diff --git a/src/paimon/core/table/system/audit_log_system_table.cpp b/src/paimon/core/table/system/audit_log_system_table.cpp index 3d3f7b3e5..7254dff60 100644 --- a/src/paimon/core/table/system/audit_log_system_table.cpp +++ b/src/paimon/core/table/system/audit_log_system_table.cpp @@ -395,7 +395,12 @@ Result> AuditLogSystemTable::NewScan( .WithStreamingMode(context->IsStreamingMode()) .WithMemoryPool(context->GetMemoryPool()) .WithExecutor(context->GetExecutor()) - .WithFileSystem(context->GetSpecificFileSystem()) + // The data table behind this view is served through the file system the view itself was + // opened with, so a context without one of its own falls back to it rather than the + // default backend. + .WithFileSystem(context->GetSpecificFileSystem() != nullptr + ? context->GetSpecificFileSystem() + : GetFileSystem()) .WithCache(context->GetCache()); if (scan_filter) { if (scan_filter->GetBucketFilter()) { @@ -458,7 +463,12 @@ Result> AuditLogSystemTable::CreateDataReadContext( .WithBranch(core_options.GetBranch()) .WithMemoryPool(context->GetMemoryPool()) .WithExecutor(context->GetExecutor()) - .WithFileSystem(context->GetSpecificFileSystem()) + // The data table behind this view is served through the file system the view itself was + // opened with, so a context without one of its own falls back to it rather than the + // default backend. + .WithFileSystem(context->GetSpecificFileSystem() != nullptr + ? context->GetSpecificFileSystem() + : GetFileSystem()) .WithFileSystemSchemeToIdentifierMap(context->GetFileSystemSchemeToIdentifierMap()) .EnablePrefetch(context->EnablePrefetch()) .SetPrefetchBatchCount(context->GetPrefetchBatchCount()) diff --git a/src/paimon/core/table/system/read_optimized_system_table.cpp b/src/paimon/core/table/system/read_optimized_system_table.cpp index 172fc964c..0d44b5989 100644 --- a/src/paimon/core/table/system/read_optimized_system_table.cpp +++ b/src/paimon/core/table/system/read_optimized_system_table.cpp @@ -71,7 +71,12 @@ Result> ReadOptimizedSystemTable::NewScan( .WithStreamingMode(context->IsStreamingMode()) .WithMemoryPool(context->GetMemoryPool()) .WithExecutor(context->GetExecutor()) - .WithFileSystem(context->GetSpecificFileSystem()) + // The data table behind this view is served through the file system the view itself was + // opened with, so a context without one of its own falls back to it rather than the + // default backend. + .WithFileSystem(context->GetSpecificFileSystem() != nullptr + ? context->GetSpecificFileSystem() + : GetFileSystem()) .WithCache(context->GetCache()); if (context->GetLimit().has_value()) { builder.SetLimit(context->GetLimit().value()); @@ -114,7 +119,12 @@ Result> ReadOptimizedSystemTable::CreateDataReadCon .SetRowToBatchThreadNumber(context->GetRowToBatchThreadNumber()) .WithMemoryPool(context->GetMemoryPool()) .WithExecutor(context->GetExecutor()) - .WithFileSystem(context->GetSpecificFileSystem()) + // The data table behind this view is served through the file system the view itself was + // opened with, so a context without one of its own falls back to it rather than the + // default backend. + .WithFileSystem(context->GetSpecificFileSystem() != nullptr + ? context->GetSpecificFileSystem() + : GetFileSystem()) .WithFileSystemSchemeToIdentifierMap(context->GetFileSystemSchemeToIdentifierMap()) .SetReadAheadCacheEnabled(context->ReadAheadCacheEnabled()) .WithCacheConfig(context->GetCacheConfig()) diff --git a/src/paimon/core/table/table_test.cpp b/src/paimon/core/table/table_test.cpp index fe99efe63..7fde349d9 100644 --- a/src/paimon/core/table/table_test.cpp +++ b/src/paimon/core/table/table_test.cpp @@ -28,6 +28,7 @@ #include "arrow/c/abi.h" #include "arrow/c/bridge.h" #include "gtest/gtest.h" +#include "paimon/cache/cache.h" #include "paimon/commit_context.h" #include "paimon/commit_message.h" #include "paimon/common/utils/checked_cast.h" @@ -72,6 +73,57 @@ Result> CreateManagedTable( /*uuid=*/"", branch); } +/// Counts the lookups a scan makes, which is enough to tell that the cache a context carries +/// reached the scan's manifest reading rather than being dropped on the way. +class CountingCache : public Cache { + public: + Result> Get( + const std::shared_ptr& key, + std::function>(const std::shared_ptr&)> + supplier) override { + gets++; + return supplier(key); + } + Status Put(const std::shared_ptr&, const std::shared_ptr&) override { + return Status::OK(); + } + void Invalidate(const std::shared_ptr&) override {} + void InvalidateAll() override {} + size_t Size() const override { + return 0; + } + + int32_t gets = 0; +}; + +/// Writes two rows through the table's own write and commit, so a scan below has data to find. +void WriteAndCommitTwoRows(const std::shared_ptr
& table, const std::string& root_path) { + WriteContextBuilder write_builder(root_path, "commit-user"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr write, + table->NewWrite(std::move(write_context))); + arrow::Int32Builder ids_builder; + ASSERT_TRUE(ids_builder.AppendValues({1, 2}).ok()); + std::shared_ptr ids_array; + ASSERT_TRUE(ids_builder.Finish(&ids_array).ok()); + auto struct_array = arrow::StructArray::Make({ids_array}, {arrow::field("id", arrow::int32(), + /*nullable=*/false)}) + .ValueOrDie(); + ::ArrowArray arrow_array; + ASSERT_TRUE(arrow::ExportArray(*struct_array, &arrow_array).ok()); + RecordBatchBuilder batch_builder(&arrow_array); + ASSERT_OK_AND_ASSIGN(std::unique_ptr batch, batch_builder.SetBucket(0).Finish()); + ASSERT_OK(write->Write(std::move(batch))); + ASSERT_OK_AND_ASSIGN(std::vector> messages, + write->PrepareCommit()); + CommitContextBuilder commit_builder(root_path, "commit-user"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, + table->NewCommit(std::move(commit_context))); + ASSERT_OK(commit->Commit(messages)); + ASSERT_OK(write->Close()); +} + } // namespace TEST(TableTest, TestTableCarriesManagersOfItsBranch) { @@ -387,10 +439,11 @@ TEST(TableTest, TestCatalogOpenedTableServesThePerTableFileSystem) { auto catalog = std::make_shared(); catalog->SetTableSchema(table->LatestSchema()); - // A catalog that hands out no per-table file system opens the table without one, and one - // that does opens it with the file system issued for that table. + // A catalog that hands out no per-table file system opens the table through the one its + // schema's options resolve - never through none, the default local one when they name no + // other - and one that does opens it with the file system issued for that table. ASSERT_OK_AND_ASSIGN(std::shared_ptr
opened, catalog->GetTable(identifier)); - EXPECT_EQ(opened->GetFileSystem(), nullptr); + EXPECT_NE(opened->GetFileSystem(), nullptr); catalog->SetTableFileSystem(dir->GetFileSystem()); ASSERT_OK_AND_ASSIGN(opened, catalog->GetTable(identifier)); @@ -723,6 +776,92 @@ TEST(TableTest, TestOperatorsRefuseAnOptionThatContradictsTheTable) { copied->NewScan(std::move(ctx))); } +TEST(TableTest, TestOperatorsIgnoreARuntimeKeyTheTableWasNotLoadedWith) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + const Identifier identifier("db", "tbl"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, + CreateManagedTable(dir->GetFileSystem(), dir->Str(), identifier, + /*branch=*/"main", + {{Options::BUCKET, "1"}, {Options::BUCKET_KEY, "id"}})); + WriteAndCommitTwoRows(table, dir->Str()); + + // `scan.mode` and `scan.snapshot-id` name no option the table was loaded with, so they name + // a runtime setting the scan does not take: the scan serves the table's own options and + // reads its latest snapshot - a merged context would have gone looking for snapshot 999, + // which was never committed. + ScanContextBuilder scan_builder(dir->Str()); + scan_builder.AddOption(Options::SCAN_MODE, "from-snapshot") + .AddOption(Options::SCAN_SNAPSHOT_ID, "999"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_ctx, scan_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, table->NewScan(std::move(scan_ctx))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); + ASSERT_FALSE(plan->Splits().empty()); + ASSERT_EQ(plan->SnapshotId(), std::optional(1)); + + // A read takes a runtime key the same way: the key names nothing the table carries, so the + // read is served rather than refused. + ReadContextBuilder read_builder(dir->Str()); + read_builder.AddOption(Options::SCAN_MODE, "from-snapshot") + .AddOption(Options::SCAN_SNAPSHOT_ID, "999"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_ctx, read_builder.Finish()); + ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr read, + table->NewRead(std::move(read_ctx))); + + // The runtime setting a caller means to serve is the copy's: `Table::Copy` derives the table + // whose options carry it, and that table's scan reads the snapshot it names - which is not + // there, as this table only ever committed snapshot 1. + ASSERT_OK_AND_ASSIGN( + std::shared_ptr
copied, + table->Copy({{Options::SCAN_MODE, "from-snapshot"}, {Options::SCAN_SNAPSHOT_ID, "999"}})); + ASSERT_OK_AND_ASSIGN(std::unique_ptr copied_scan_ctx, + ScanContextBuilder(dir->Str()).Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr copied_scan, + copied->NewScan(std::move(copied_scan_ctx))); + ASSERT_NOK_WITH_MSG(copied_scan->CreatePlan(), "snapshot"); +} + +TEST(TableTest, TestCreateFailsWhenTheSchemaOptionsDoNotParse) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + // The schema itself carries no validation, so a schema written with options no + // `CoreOptions` accepts surfaces when the store parses them - at the load, not at the first + // operator built on the table. + ASSERT_OK_AND_ASSIGN( + std::unique_ptr created, + TableSchema::Create( + /*schema_id=*/0, + arrow::schema({arrow::field("id", arrow::int32(), + /*nullable=*/false)}), + /*partition_keys=*/{}, /*primary_keys=*/{}, {{Options::TARGET_FILE_ROW_NUM, "0"}})); + auto schema = checked_pointer_cast(std::shared_ptr(std::move(created))); + ASSERT_NOK_WITH_MSG( + FileStoreTable::Create(dir->GetFileSystem(), dir->Str(), schema, Identifier("db", "tbl")), + "target-file-row-num should be at least 1"); +} + +TEST(TableTest, TestTheCacheAScanContextCarriesReachesTheScan) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + const Identifier identifier("db", "tbl"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, + CreateManagedTable(dir->GetFileSystem(), dir->Str(), identifier, + /*branch=*/"main", + {{Options::BUCKET, "1"}, {Options::BUCKET_KEY, "id"}})); + WriteAndCommitTwoRows(table, dir->Str()); + + // The cache is a runtime setting the scan serves from the context rather than from the + // table's own options, so a scan built with one reads its manifests through it. + auto cache = std::make_shared(); + ScanContextBuilder scan_builder(dir->Str()); + scan_builder.WithCache(cache); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_ctx, scan_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, table->NewScan(std::move(scan_ctx))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); + ASSERT_FALSE(plan->Splits().empty()); + ASSERT_GT(cache->gets, 0) << "the cache the scan context carries never reached the scan"; +} + TEST(TableTest, TestCopyServesTheTableWithTheGivenOptionsOnTop) { auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); diff --git a/src/paimon/rest/rest_catalog.cpp b/src/paimon/rest/rest_catalog.cpp index b7dcfe04f..083fb62d5 100644 --- a/src/paimon/rest/rest_catalog.cpp +++ b/src/paimon/rest/rest_catalog.cpp @@ -434,6 +434,24 @@ Result> RestCatalog::LoadSystemTable( } Result> RestCatalog::GetTable(const Identifier& identifier) const { + // The global system tables of the "sys" database expose the catalog itself rather than any + // table the server stands behind, so they are served locally - the way `LoadTableSchema` + // serves them their schema - and never reach the managed-table load below, whose schema + // cast a read-only system table schema would not survive. + if (CatalogUtils::IsSystemDatabase(identifier.GetDatabaseName())) { + PAIMON_ASSIGN_OR_RAISE(bool supported, + GlobalSystemTableLoader::IsSupported(identifier.GetObjectName(), + api_->GetMergedOptions())); + if (!supported) { + return Status::NotExist(fmt::format("{} not exist", identifier.ToString())); + } + GlobalSystemTableContext context; + context.catalog = this; + context.fs = fs_; + context.warehouse = warehouse_; + context.catalog_options = api_->GetMergedOptions(); + return GlobalSystemTableLoader::Load(identifier.GetObjectName(), context); + } // A `T$system` identifier opens the read-only view it names, which reads the data table it // belongs to and answers for itself through the same `Table` interface. PAIMON_ASSIGN_OR_RAISE(bool is_system_table, identifier.IsSystemTable()); diff --git a/test/inte/scan_and_read_inte_test.cpp b/test/inte/scan_and_read_inte_test.cpp index 41d35de8d..4fa4e1da1 100644 --- a/test/inte/scan_and_read_inte_test.cpp +++ b/test/inte/scan_and_read_inte_test.cpp @@ -813,8 +813,17 @@ TEST_P(ScanAndReadInteTest, TestWithAppendBucketKeyPointLookup) { auto lookup_predicate = PredicateBuilder::Equal( 0, "rowkey", field_type, Literal(field_type, lookup_key.data(), lookup_key.size())); + // The rescaled bucket count and, with it, the cache settings `FinishScanContext` + // also puts on the context: a table already held takes its runtime settings + // through the copy it is made of, and the context carries them only to agree + // with it. + std::map dynamic_options{ + {Options::BUCKET, current_bucket_count}}; + if (EnableSnapshotLiveManifestCache()) { + dynamic_options.emplace(Options::SCAN_MANIFEST_ENTRY_CACHE_MAX_SNAPSHOTS, "3"); + } ASSERT_OK_AND_ASSIGN(std::shared_ptr
rescaled_table, - table->Copy({{Options::BUCKET, current_bucket_count}})); + table->Copy(dynamic_options)); ScanContextBuilder scan_builder(table_path); scan_builder.SetPredicate(lookup_predicate) .AddOption(Options::BUCKET, current_bucket_count); @@ -4065,8 +4074,14 @@ TEST_P(ScanAndReadInteTest, TestWithPKBucketSelectByPredicate) { catalog->GetTable(Identifier("pk_table_scan_and_read_dv", "pk_table_scan_and_read_dv"))); for (const std::string current_bucket_count : {"2", "4", "8", "17"}) { SCOPED_TRACE(current_bucket_count); - ASSERT_OK_AND_ASSIGN(std::shared_ptr
rescaled_table, - table->Copy({{Options::BUCKET, current_bucket_count}})); + // The rescaled bucket count and, with it, the cache settings `FinishScanContext` also + // puts on the context: a table already held takes its runtime settings through the copy + // it is made of, and the context carries them only to agree with it. + std::map dynamic_options{{Options::BUCKET, current_bucket_count}}; + if (EnableSnapshotLiveManifestCache()) { + dynamic_options.emplace(Options::SCAN_MANIFEST_ENTRY_CACHE_MAX_SNAPSHOTS, "3"); + } + ASSERT_OK_AND_ASSIGN(std::shared_ptr
rescaled_table, table->Copy(dynamic_options)); ScanContextBuilder scan_context_builder(table_path); scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "6"); scan_context_builder.AddOption(Options::BUCKET, current_bucket_count); From 472c9c5f1c8b8df47a650103c52abef0f1e1ccf5 Mon Sep 17 00:00:00 2001 From: "yonghao.fyh" Date: Fri, 23 Oct 2026 14:23:42 +0800 Subject: [PATCH 07/17] fix --- .../core/operation/internal_read_context.cpp | 4 +- src/paimon/core/table/file_store.cpp | 16 ++-- src/paimon/core/table/file_store.h | 21 ++++- src/paimon/core/table/file_store_table.cpp | 87 +++++++++++++++---- src/paimon/core/table/file_store_table.h | 6 +- .../table/system/audit_log_system_table.cpp | 15 ++-- .../system/read_optimized_system_table.cpp | 15 ++-- src/paimon/rest/rest_catalog.cpp | 14 +-- 8 files changed, 131 insertions(+), 47 deletions(-) diff --git a/src/paimon/core/operation/internal_read_context.cpp b/src/paimon/core/operation/internal_read_context.cpp index 778976a46..784123828 100644 --- a/src/paimon/core/operation/internal_read_context.cpp +++ b/src/paimon/core/operation/internal_read_context.cpp @@ -290,8 +290,8 @@ Result> InternalReadContext::Create( // resolves its own. PAIMON_ASSIGN_OR_RAISE( CoreOptions core_options, - CoreOptions::FromMap(options, file_system != nullptr ? file_system - : context->GetSpecificFileSystem())); + CoreOptions::FromMap( + options, file_system != nullptr ? file_system : context->GetSpecificFileSystem())); core_options.WithCache(context->GetCache()); return CreateFromCoreOptions(context, table_schema, std::move(core_options), path); } diff --git a/src/paimon/core/table/file_store.cpp b/src/paimon/core/table/file_store.cpp index 2c5a19d96..472626a01 100644 --- a/src/paimon/core/table/file_store.cpp +++ b/src/paimon/core/table/file_store.cpp @@ -51,13 +51,15 @@ Result> FileStore::Create( const std::shared_ptr& file_system, const std::string& root_path, const std::shared_ptr& schema, const std::string& branch, const std::shared_ptr& schema_manager, - const std::shared_ptr& snapshot_manager) { - auto store = std::shared_ptr( - new FileStore(file_system, root_path, schema, branch, schema_manager, snapshot_manager)); + const std::shared_ptr& snapshot_manager, + const std::map& fs_scheme_map) { + auto store = std::shared_ptr(new FileStore( + file_system, root_path, schema, branch, schema_manager, snapshot_manager, fs_scheme_map)); if (schema != nullptr) { // The options the schema was loaded with, on the branch this store serves: what reads // through them - whose cache keys are scoped by the branch - reads the same one the - // managers point at. + // managers point at. The scheme map is the one the schema was read through, so a scheme + // it mapped resolves here the way it resolved there. std::map options = schema->Options(); if (BranchManager::IsMainBranch(store->branch_)) { options.erase(Options::BRANCH); @@ -65,7 +67,7 @@ Result> FileStore::Create( options[Options::BRANCH] = store->branch_; } PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, - CoreOptions::FromMap(options, file_system, {})); + CoreOptions::FromMap(options, file_system, fs_scheme_map)); store->core_options_ = std::move(core_options); } // The file system the table is served through: the one the options resolve - a schema @@ -98,9 +100,11 @@ Result> FileStore::Create( FileStore::FileStore(const std::shared_ptr& file_system, const std::string& root_path, const std::shared_ptr& schema, const std::string& branch, const std::shared_ptr& schema_manager, - const std::shared_ptr& snapshot_manager) + const std::shared_ptr& snapshot_manager, + const std::map& fs_scheme_map) : file_system_(file_system), specified_file_system_(file_system), + fs_scheme_map_(fs_scheme_map), root_path_(root_path), schema_(schema), branch_(BranchManager::NormalizeBranch(branch)), diff --git a/src/paimon/core/table/file_store.h b/src/paimon/core/table/file_store.h index 6c2396bd6..66ccc09c2 100644 --- a/src/paimon/core/table/file_store.h +++ b/src/paimon/core/table/file_store.h @@ -19,6 +19,7 @@ #pragma once +#include #include #include #include @@ -77,6 +78,11 @@ class FileStore { /// @param snapshot_manager A snapshot manager to carry instead of building one. /// A null manager is built over `root_path` on `branch`, so the operators always read /// through the managers the store carries. + /// @param fs_scheme_map The scheme-to-identifier map the caller's contexts resolve custom + /// schemes through. It takes part in building the store's options the way it took + /// part in reading the schema: without it a schema naming a custom scheme would + /// resolve to the default backend once its operators run. It is kept for a copy to + /// hand over again, so the copy resolves as this store does. /// /// Building the store parses the schema's options, so it fails when they do not parse: the /// error is the caller's to see, which is why a store is built through this factory rather @@ -85,7 +91,8 @@ class FileStore { const std::shared_ptr& file_system, const std::string& root_path, const std::shared_ptr& schema, const std::string& branch, const std::shared_ptr& schema_manager = nullptr, - const std::shared_ptr& snapshot_manager = nullptr); + const std::shared_ptr& snapshot_manager = nullptr, + const std::map& fs_scheme_map = {}); /// The branch this store serves, as resolved when it was built. const std::string& branch() const { @@ -125,6 +132,12 @@ class FileStore { return specified_file_system_; } + /// The scheme-to-identifier map the options were resolved with, kept for a copy to resolve + /// through again: see `Create`'s `fs_scheme_map`. + const std::map& fs_scheme_map() const { + return fs_scheme_map_; + } + /// The root path of the table in its file system, as the table was opened with it. const std::string& root_path() const { return root_path_; @@ -199,13 +212,17 @@ class FileStore { FileStore(const std::shared_ptr& file_system, const std::string& root_path, const std::shared_ptr& schema, const std::string& branch, const std::shared_ptr& schema_manager, - const std::shared_ptr& snapshot_manager); + const std::shared_ptr& snapshot_manager, + const std::map& fs_scheme_map); std::shared_ptr file_system_; /// The file system the caller opened the table through, as handed to `Create` - null when /// the call had none - kept for a copy to hand over again: what `file_system_` holds is /// resolved, and would pin the copy to this table's resolution. std::shared_ptr specified_file_system_; + /// The scheme-to-identifier map the options were resolved with, kept for a copy to resolve + /// through again. + std::map fs_scheme_map_; std::string root_path_; std::shared_ptr schema_; std::string branch_; diff --git a/src/paimon/core/table/file_store_table.cpp b/src/paimon/core/table/file_store_table.cpp index 2e5f77af6..e46259345 100644 --- a/src/paimon/core/table/file_store_table.cpp +++ b/src/paimon/core/table/file_store_table.cpp @@ -36,6 +36,7 @@ #include "paimon/common/utils/checked_cast.h" #include "paimon/common/utils/options_utils.h" #include "paimon/common/utils/path_util.h" +#include "paimon/common/utils/sensitive_config_utils.h" #include "paimon/common/utils/string_utils.h" #include "paimon/core/catalog/catalog_snapshot_commit.h" #include "paimon/core/catalog/catalog_utils.h" @@ -141,6 +142,26 @@ Result> TryLoadFormatTable( /*location_carries_paimon_metadata=*/true, options); } +/// Whether an option of a copy names the file system its store resolves: the backend +/// `file-system` picks, or the access configuration a `fs.` key carries. A copy whose dynamic +/// options change one of these resolves to another file system, so the managers of the table it +/// was copied from - built over that table's resolution - are not its to inherit: they would +/// keep reading metadata through the backend the copy just left. +bool FileSystemResolutionMayChange(const TableSchema& schema, + const std::map& dynamic_options) { + const std::map& schema_options = schema.Options(); + for (const auto& [key, value] : dynamic_options) { + if (key != Options::FILE_SYSTEM && key.rfind("fs.", 0) != 0) { + continue; + } + auto schema_option = schema_options.find(key); + if (schema_option == schema_options.end() || schema_option->second != value) { + return true; + } + } + return false; +} + /// The options a context carries, merged into the table the load hands back: a table is the /// product of the load, so the options speak there - the way `Copy` merges them, over the ones /// the schema carries - and `NewXxx` on the product serves them without merging again. A @@ -183,7 +204,7 @@ Result> TableOfLoadedSchema( const std::string& branch, const std::map& options, const std::shared_ptr& schema_manager, const std::shared_ptr& latest_schema, const std::optional& identifier, - const std::string& uuid = "") { + const std::string& uuid = "", const std::map& fs_scheme_map = {}) { PAIMON_ASSIGN_OR_RAISE(std::shared_ptr format_table, TryLoadFormatTable(file_system, path, branch, options, latest_schema)); if (format_table != nullptr) { @@ -195,7 +216,8 @@ Result> TableOfLoadedSchema( identifier.value_or(Identifier(Identifier::kUnknownDatabase, PathUtil::GetName(path))); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table, FileStoreTable::Create(explicit_file_system, path, latest_schema, - table_identifier, uuid, branch, schema_manager)); + table_identifier, uuid, branch, schema_manager, + /*snapshot_manager=*/nullptr, fs_scheme_map)); return WithContextOptions(table, latest_schema, options); } @@ -213,7 +235,7 @@ Result> ManagedTableOf( const std::shared_ptr& explicit_file_system, const std::string& path, const std::string& branch, const std::map& options, const std::optional& schema_json, const std::optional& identifier, - const std::string& uuid = "") { + const std::string& uuid = "", const std::map& fs_scheme_map = {}) { auto schema_manager = std::make_shared(file_system, path, branch); std::shared_ptr latest_schema; if (branch == BranchManager::DEFAULT_MAIN_BRANCH && schema_json) { @@ -228,7 +250,7 @@ Result> ManagedTableOf( } } return TableOfLoadedSchema(file_system, explicit_file_system, path, branch, options, - schema_manager, latest_schema, identifier, uuid); + schema_manager, latest_schema, identifier, uuid, fs_scheme_map); } /// The identifier a catalog knows the table by, on `branch`: a non-main branch is folded into @@ -325,9 +347,12 @@ Result> TableOf(const ReadContext& ctx) { // which files there are and how a row is put back together is its own. How a file is opened // is not: both paths go through `DataFileReaderFactory`. const std::string branch = ctx.GetBranch(); + // The scheme map the schema was read through, so the table it describes resolves custom + // schemes the same way - the data belongs to the metadata, both sit behind the map. return ManagedTableOf(tmp_core_options.GetFileSystem(), ctx.GetSpecificFileSystem(), ctx.GetPath(), branch, ctx.GetOptions(), ctx.GetSpecificTableSchema(), - ctx.GetIdentifier()); + ctx.GetIdentifier(), /*uuid=*/"", + ctx.GetFileSystemSchemeToIdentifierMap()); } /// The table a write context names, read from what the context carries: a format table the @@ -365,7 +390,8 @@ Result> TableOf(const WriteContext& ctx) { // A format table writes plain data files into a directory, so it never reaches the manifest // path below. return ManagedTableOf(tmp_options.GetFileSystem(), specific_fs, ctx.GetRootPath(), branch, - ctx.GetOptions(), catalog_table_schema, ctx.GetIdentifier()); + ctx.GetOptions(), catalog_table_schema, ctx.GetIdentifier(), /*uuid=*/"", + ctx.GetFileSystemSchemeToIdentifierMap()); } /// The table a commit context names, read from what the context carries: a format table the @@ -425,9 +451,12 @@ Status CheckOptionsAgreeWithTable(const TableSchema& table_schema, for (const auto& [key, value] : context_options) { auto table_option = table_options.find(key); if (table_option != table_options.end() && table_option->second != value) { + // A value may be a credential, and the error travels into logs: it is redacted the + // way every surface that exposes options redacts them. return Status::Invalid( - fmt::format("option '{}' is '{}' in the context but '{}' on the table", key, value, - table_option->second)); + fmt::format("option '{}' is '{}' in the context but '{}' on the table", key, + SensitiveConfigUtils::RedactValue(key, value), + SensitiveConfigUtils::RedactValue(key, table_option->second))); } } return Status::OK(); @@ -470,7 +499,8 @@ Result> FileStoreTable::Create( const std::shared_ptr& file_system, const std::string& root_path, const std::shared_ptr& schema, const Identifier& identifier, const std::string& uuid, const std::string& branch, const std::shared_ptr& schema_manager, - const std::shared_ptr& snapshot_manager) { + const std::shared_ptr& snapshot_manager, + const std::map& fs_scheme_map) { // The kind the schema's primary keys name, which no schema change can add or drop, so the // kind the table is built as is the kind every schema its root path ever held describes. A // caller without a schema gets the append-only kind, whose operators then report that no @@ -488,7 +518,7 @@ Result> FileStoreTable::Create( // here: the table it names cannot be served, not by one operator more than another. PAIMON_ASSIGN_OR_RAISE(std::shared_ptr store, FileStore::Create(file_system, root_path, table_schema, branch, - schema_manager, snapshot_manager)); + schema_manager, snapshot_manager, fs_scheme_map)); if (table_schema != nullptr && !table_schema->PrimaryKeys().empty()) { return std::make_shared(store, identifier, uuid); } @@ -507,15 +537,22 @@ Result> FileStoreTable::Copy( overridable.erase(Options::TYPE); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr merged_schema, loaded_schema->MergeOptions(overridable)); - // The managers are handed over, not rebuilt: the copy serves the same root path and branch, - // so its operators read the schema cache this table already warmed. The file system is the - // one the caller opened this table through, not the one it resolved to: the copy resolves - // from its own merged options again, so a file-system option among the ones it was given - // speaks rather than being pinned to this table's resolution. + // The managers are handed over, not rebuilt, when the copy resolves the same file system + // this table does: its operators then read the schema cache this table already warmed. The + // file system is the one the caller opened this table through, not the one it resolved to: + // the copy resolves from its own merged options again, so a file-system option among the + // ones it was given speaks rather than being pinned to this table's resolution - and when + // one does change the resolution, fresh managers are built over it instead, so the metadata + // is read through the same backend the data is. The scheme map travels the same way, so the + // copy resolves custom schemes as this table does. + const bool fs_resolution_may_change = + FileSystemResolutionMayChange(*loaded_schema, overridable); return FileStoreTable::Create(store_->specified_file_system(), root_path_, std::shared_ptr(std::move(merged_schema)), identifier_, CatalogUuid().value_or(""), CurrentBranch(), - GetSchemaManager(), GetSnapshotManager()); + fs_resolution_may_change ? nullptr : GetSchemaManager(), + fs_resolution_may_change ? nullptr : GetSnapshotManager(), + store_->fs_scheme_map()); } Result> FileStoreTable::SwitchToBranch(const std::string& branch) const { @@ -525,7 +562,8 @@ Result> FileStoreTable::SwitchToBranch(const std::string& // the same table to the caller. return FileStoreTable::Create(store_->specified_file_system(), root_path_, schema_, identifier_, CatalogUuid().value_or(""), CurrentBranch(), - GetSchemaManager(), GetSnapshotManager()); + GetSchemaManager(), GetSnapshotManager(), + store_->fs_scheme_map()); } // The schema of the branch switched to, read from its own directory, which a branch without // one does not have. @@ -546,7 +584,8 @@ Result> FileStoreTable::SwitchToBranch(const std::string& return FileStoreTable::Create(store_->specified_file_system(), root_path_, std::shared_ptr(std::move(branch_schema)), identifier_, CatalogUuid().value_or(""), target_branch, - schema_manager); + schema_manager, /*snapshot_manager=*/nullptr, + store_->fs_scheme_map()); } Status FileStoreTable::RestoreRealtimeCommittedProgress( @@ -838,8 +877,20 @@ Result> CreateFallbackReadContext( } assert(table_schema); auto options = table_schema->Options(); + // The dynamic options the load merged into the table this read serves - a sequence-number + // flag, a batch size - speak for the read, and the fallback branch serves the same read: + // they are merged over the fallback branch's own schema options the way the load merged + // them over the table's, so the fallback reader recognises what the main one does. `type` + // stays the branch's own: a copy never restyles the table, and neither does a fallback. + for (const auto& [key, value] : context->GetOptions()) { + if (key != Options::TYPE) { + options[key] = value; + } + } if (branch != BranchManager::DEFAULT_MAIN_BRANCH) { options[Options::BRANCH] = branch; + } else { + options.erase(Options::BRANCH); } return InternalReadContext::Create(context, table_schema, options, file_system, &root_path); } diff --git a/src/paimon/core/table/file_store_table.h b/src/paimon/core/table/file_store_table.h index 8df9c4d86..df841d548 100644 --- a/src/paimon/core/table/file_store_table.h +++ b/src/paimon/core/table/file_store_table.h @@ -78,13 +78,17 @@ class FileStoreTable : public Table { /// `Copy` hands its table's over so both share one. /// A null manager is built over `root_path` on `branch`, so the operators always read /// through the managers the table carries. + /// @param fs_scheme_map The scheme-to-identifier map the caller's contexts resolve custom + /// schemes through, which the store's options resolve through as well - see + /// `FileStore::Create`. `Copy` hands its table's map over. static Result> Create( const std::shared_ptr& file_system, const std::string& root_path, const std::shared_ptr& schema, const Identifier& identifier, const std::string& uuid = "", const std::string& branch = BranchManager::DEFAULT_MAIN_BRANCH, const std::shared_ptr& schema_manager = nullptr, - const std::shared_ptr& snapshot_manager = nullptr); + const std::shared_ptr& snapshot_manager = nullptr, + const std::map& fs_scheme_map = {}); /// The branch this table serves, as resolved when the table was built: the schema the table /// was loaded with came from it, and its managers point at it. diff --git a/src/paimon/core/table/system/audit_log_system_table.cpp b/src/paimon/core/table/system/audit_log_system_table.cpp index 7254dff60..e191b653a 100644 --- a/src/paimon/core/table/system/audit_log_system_table.cpp +++ b/src/paimon/core/table/system/audit_log_system_table.cpp @@ -458,17 +458,20 @@ Result> AuditLogSystemTable::CreateDataReadContext( PAIMON_ASSIGN_OR_RAISE(std::shared_ptr base_read_schema, BaseReadSchema()); PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, CoreOptions::FromMap(read_options)); ReadContextBuilder builder(table_path_); + // The data table behind this view is served through the file system the view itself was + // opened with, but only when the context names neither a file system of its own nor a + // scheme map to resolve through - a file system and a scheme map cannot both be named, and + // the context's own answer always wins. + std::shared_ptr data_table_fs = context->GetSpecificFileSystem(); + if (data_table_fs == nullptr && context->GetFileSystemSchemeToIdentifierMap().empty()) { + data_table_fs = GetFileSystem(); + } builder.SetOptions(read_options) .SetReadFieldNames(base_read_schema->field_names()) .WithBranch(core_options.GetBranch()) .WithMemoryPool(context->GetMemoryPool()) .WithExecutor(context->GetExecutor()) - // The data table behind this view is served through the file system the view itself was - // opened with, so a context without one of its own falls back to it rather than the - // default backend. - .WithFileSystem(context->GetSpecificFileSystem() != nullptr - ? context->GetSpecificFileSystem() - : GetFileSystem()) + .WithFileSystem(data_table_fs) .WithFileSystemSchemeToIdentifierMap(context->GetFileSystemSchemeToIdentifierMap()) .EnablePrefetch(context->EnablePrefetch()) .SetPrefetchBatchCount(context->GetPrefetchBatchCount()) diff --git a/src/paimon/core/table/system/read_optimized_system_table.cpp b/src/paimon/core/table/system/read_optimized_system_table.cpp index 0d44b5989..d7193b5cb 100644 --- a/src/paimon/core/table/system/read_optimized_system_table.cpp +++ b/src/paimon/core/table/system/read_optimized_system_table.cpp @@ -108,6 +108,14 @@ Result> ReadOptimizedSystemTable::CreateDataReadCon branch = branch_iter->second; } ReadContextBuilder builder(table_path_); + // The data table behind this view is served through the file system the view itself was + // opened with, but only when the context names neither a file system of its own nor a + // scheme map to resolve through - a file system and a scheme map cannot both be named, and + // the context's own answer always wins. + std::shared_ptr data_table_fs = context->GetSpecificFileSystem(); + if (data_table_fs == nullptr && context->GetFileSystemSchemeToIdentifierMap().empty()) { + data_table_fs = GetFileSystem(); + } builder.SetOptions(options) .WithBranch(branch) .SetPredicate(context->GetPredicate()) @@ -119,12 +127,7 @@ Result> ReadOptimizedSystemTable::CreateDataReadCon .SetRowToBatchThreadNumber(context->GetRowToBatchThreadNumber()) .WithMemoryPool(context->GetMemoryPool()) .WithExecutor(context->GetExecutor()) - // The data table behind this view is served through the file system the view itself was - // opened with, so a context without one of its own falls back to it rather than the - // default backend. - .WithFileSystem(context->GetSpecificFileSystem() != nullptr - ? context->GetSpecificFileSystem() - : GetFileSystem()) + .WithFileSystem(data_table_fs) .WithFileSystemSchemeToIdentifierMap(context->GetFileSystemSchemeToIdentifierMap()) .SetReadAheadCacheEnabled(context->ReadAheadCacheEnabled()) .WithCacheConfig(context->GetCacheConfig()) diff --git a/src/paimon/rest/rest_catalog.cpp b/src/paimon/rest/rest_catalog.cpp index 083fb62d5..07e4b000b 100644 --- a/src/paimon/rest/rest_catalog.cpp +++ b/src/paimon/rest/rest_catalog.cpp @@ -425,12 +425,14 @@ Result> RestCatalog::LoadSystemTable( if (branch) { dynamic_options[Options::BRANCH] = branch.value(); } - // The file system is only used to build the system table, whose schema does not - // read any file, so the catalog wide one is enough here. The credentials of the - // table are applied when its rows are read, through the file system of the read or - // scan context. - return SystemTableLoader::Load(system_table_name.value(), fs_, data_table_path, latest_schema, - dynamic_options); + // The system table reads the data table's own files - snapshots, manifests, data - so it is + // served by the file system that table's credentials open, which with data tokens is the + // only one that can: `GetTableFileSystem` folds the system identifier to the data table it + // belongs to and hands the catalog-wide file system out when no tokens are issued. + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table_file_system, + GetTableFileSystem(identifier)); + return SystemTableLoader::Load(system_table_name.value(), table_file_system, data_table_path, + latest_schema, dynamic_options); } Result> RestCatalog::GetTable(const Identifier& identifier) const { From c4f9d468e3a6664fe631f1f705eb674ac80071cb Mon Sep 17 00:00:00 2001 From: "yonghao.fyh" Date: Fri, 23 Oct 2026 14:42:48 +0800 Subject: [PATCH 08/17] fix --- src/paimon/core/table/file_store_table.cpp | 7 +++- src/paimon/core/table/table_test.cpp | 45 ++++++++++++++++++++++ 2 files changed, 51 insertions(+), 1 deletion(-) diff --git a/src/paimon/core/table/file_store_table.cpp b/src/paimon/core/table/file_store_table.cpp index e46259345..11efaf24d 100644 --- a/src/paimon/core/table/file_store_table.cpp +++ b/src/paimon/core/table/file_store_table.cpp @@ -532,9 +532,14 @@ Result> FileStoreTable::Copy( return Status::Invalid(fmt::format("not found latest schema of {}", FullName())); } // `type` is structural and read from the schema alone, the promise `Copy` makes: an option - // given at the call must not restyle the table. + // given at the call must not restyle the table. `branch` is the same kind of promise: the + // copy is the table it was copied from, on the branch that table serves, so a branch option + // given at the call moves neither - it would only claim a branch the copy does not serve - + // and a caller that means to move it has `SwitchToBranch`. Both are dropped before the + // merge, so the copy's options never speak of a branch its operators do not. std::map overridable = dynamic_options; overridable.erase(Options::TYPE); + overridable.erase(Options::BRANCH); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr merged_schema, loaded_schema->MergeOptions(overridable)); // The managers are handed over, not rebuilt, when the copy resolves the same file system diff --git a/src/paimon/core/table/table_test.cpp b/src/paimon/core/table/table_test.cpp index 7fde349d9..b4478918b 100644 --- a/src/paimon/core/table/table_test.cpp +++ b/src/paimon/core/table/table_test.cpp @@ -153,6 +153,51 @@ TEST(TableTest, TestTableCarriesManagersOfItsBranch) { EXPECT_EQ(file_store_copy->GetSnapshotManager(), file_store_table->GetSnapshotManager()); } +TEST(TableTest, TestCopyKeepsTheBranchTheTableServes) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + const Identifier identifier("db", "tbl"); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr
table, + CreateManagedTable(dir->GetFileSystem(), dir->Str(), identifier, /*branch=*/"main", + {{Options::BUCKET, "1"}, {Options::BUCKET_KEY, "id"}})); + + // A copy handed a `branch` option is the table it was copied from: the branch is the + // table's own - `SwitchToBranch` is the way to change it - so the option moves neither the + // copy nor its options, and an operator asked for the branch the copy serves - `main` - is + // served rather than refused by options claiming a `dev` the copy does not serve. + ASSERT_OK_AND_ASSIGN(std::shared_ptr
copy, table->Copy({{Options::BRANCH, "dev"}})); + auto file_store_copy = std::dynamic_pointer_cast(copy); + ASSERT_TRUE(file_store_copy != nullptr); + EXPECT_EQ(file_store_copy->CurrentBranch(), "main"); + { + ScanContextBuilder scan_builder(dir->Str()); + scan_builder.AddOption(Options::BRANCH, "main"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, scan_builder.Finish()); + ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr scan, + file_store_copy->NewScan(std::move(ctx))); + } + + // The same the other way round: a copy of a branch table serves that branch still, so + // naming the branch it serves - `dev` - is agreed with rather than refused. + ASSERT_OK_AND_ASSIGN(std::shared_ptr
branch_table, + CreateManagedTable(dir->GetFileSystem(), dir->Str() + "/branch_tbl", + identifier, /*branch=*/"dev", + {{Options::BUCKET, "1"}, {Options::BUCKET_KEY, "id"}})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
branch_copy, + branch_table->Copy({{Options::BRANCH, "main"}})); + auto branch_file_store_copy = std::dynamic_pointer_cast(branch_copy); + ASSERT_TRUE(branch_file_store_copy != nullptr); + EXPECT_EQ(branch_file_store_copy->CurrentBranch(), "dev"); + { + ScanContextBuilder scan_builder(dir->Str() + "/branch_tbl"); + scan_builder.AddOption(Options::BRANCH, "dev"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, scan_builder.Finish()); + ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr scan, + branch_file_store_copy->NewScan(std::move(ctx))); + } +} + TEST(TableTest, TestSwitchToBranch) { auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); From 7b3503290e1c5aee07c67fe8a1e3b4b00175d222 Mon Sep 17 00:00:00 2001 From: "yonghao.fyh" Date: Fri, 23 Oct 2026 15:03:31 +0800 Subject: [PATCH 09/17] fix --- src/paimon/core/operation/internal_read_context.cpp | 2 +- src/paimon/core/table/file_store_table.cpp | 2 +- src/paimon/core/table/format/format_table.cpp | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/paimon/core/operation/internal_read_context.cpp b/src/paimon/core/operation/internal_read_context.cpp index 784123828..3eb169ac7 100644 --- a/src/paimon/core/operation/internal_read_context.cpp +++ b/src/paimon/core/operation/internal_read_context.cpp @@ -413,7 +413,7 @@ Result> InternalReadContext::CreateFromCore context, table_schema, read_schema, core_options, arrow_pool, path != nullptr ? std::optional(*path) : std::nullopt)); PAIMON_RETURN_NOT_OK(internal_context->BindIndexScores()); - return internal_context + return internal_context; } Status InternalReadContext::BindIndexScores() { diff --git a/src/paimon/core/table/file_store_table.cpp b/src/paimon/core/table/file_store_table.cpp index 11efaf24d..c8e50eb62 100644 --- a/src/paimon/core/table/file_store_table.cpp +++ b/src/paimon/core/table/file_store_table.cpp @@ -329,7 +329,7 @@ Result> TableOf(const ReadContext& ctx) { PAIMON_ASSIGN_OR_RAISE(std::optional system_table_path, SystemTableLoader::TryParsePath(ctx.GetPath())); if (system_table_path) { - if (PredicateUtils::ContainsFullTextMatch(context->GetPredicate())) { + if (PredicateUtils::ContainsFullTextMatch(ctx.GetPredicate())) { return Status::NotImplemented( "FullTextMatch predicate execution is not implemented yet"); } diff --git a/src/paimon/core/table/format/format_table.cpp b/src/paimon/core/table/format/format_table.cpp index 6f2ac16a4..cb4141dde 100644 --- a/src/paimon/core/table/format/format_table.cpp +++ b/src/paimon/core/table/format/format_table.cpp @@ -43,6 +43,7 @@ #include "paimon/file_store_commit.h" #include "paimon/file_store_write.h" #include "paimon/fs/file_system.h" +#include "paimon/predicate/predicate_utils.h" #include "paimon/read_context.h" #include "paimon/scan_context.h" #include "paimon/status.h" From d6cbfe387748e0512a02766183ed6f5f74183cbe Mon Sep 17 00:00:00 2001 From: "yonghao.fyh" Date: Fri, 23 Oct 2026 15:30:38 +0800 Subject: [PATCH 10/17] fix --- .../core/table/source/table_read_test.cpp | 20 +++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/src/paimon/core/table/source/table_read_test.cpp b/src/paimon/core/table/source/table_read_test.cpp index b6aef3992..072ba3e1d 100644 --- a/src/paimon/core/table/source/table_read_test.cpp +++ b/src/paimon/core/table/source/table_read_test.cpp @@ -36,6 +36,7 @@ #include "paimon/core/operation/split_read.h" #include "paimon/core/schema/schema_manager.h" #include "paimon/core/schema/table_schema.h" +#include "paimon/core/table/file_store_table.h" #include "paimon/core/table/source/append_only_table_read.h" #include "paimon/core/table/source/data_split_impl.h" #include "paimon/core/table/source/key_value_table_read.h" @@ -134,20 +135,22 @@ TEST(TableReadTest, TestFullTextMatchExecutionIsNotImplemented) { auto fs = std::make_shared(); SchemaManager schema_manager(fs, table_path); ASSERT_OK_AND_ASSIGN(std::shared_ptr schema, schema_manager.ReadSchema(0)); - Table table(fs, table_path, schema, Identifier("append_09.db", "append_09")); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr table, + FileStoreTable::Create(fs, table_path, schema, Identifier("append_09.db", "append_09"))); for (bool table_entry : {false, true}) { ReadContextBuilder builder(table_path); builder.SetReadFieldNames({"f0", "f1"}).SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(auto context, builder.Finish()); auto result = - table_entry ? table.NewRead(std::move(context)) : TableRead::Create(std::move(context)); + table_entry ? table->NewRead(std::move(context)) : TableRead::Create(std::move(context)); ASSERT_NOK_WITH_MSG(result, "FullTextMatch predicate execution is not implemented yet"); ASSERT_TRUE(result.status().IsNotImplemented()); builder.SetReadFieldNames({"f0", "f1", "_f0_INDEX_SCORE"}).SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(context, builder.Finish()); result = - table_entry ? table.NewRead(std::move(context)) : TableRead::Create(std::move(context)); + table_entry ? table->NewRead(std::move(context)) : TableRead::Create(std::move(context)); ASSERT_NOK_WITH_MSG(result, "Score-producing FullTextMatch cannot appear under OR"); } } @@ -159,13 +162,15 @@ TEST(TableReadTest, TestPerFieldScoreExecutionIsNotImplemented) { auto fs = std::make_shared(); SchemaManager schema_manager(fs, path); ASSERT_OK_AND_ASSIGN(std::shared_ptr schema, schema_manager.ReadSchema(0)); - Table table(fs, path, schema, Identifier("append_09.db", "append_09")); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr table, + FileStoreTable::Create(fs, path, schema, Identifier("append_09.db", "append_09"))); for (bool table_entry : {false, true}) { ReadContextBuilder builder(path); builder.SetFullTextSearch(search).SetReadFieldNames({"f1", "_f0_INDEX_SCORE"}); ASSERT_OK_AND_ASSIGN(auto context, builder.Finish()); auto result = - table_entry ? table.NewRead(std::move(context)) : TableRead::Create(std::move(context)); + table_entry ? table->NewRead(std::move(context)) : TableRead::Create(std::move(context)); ASSERT_NOK_WITH_MSG(result, "Per-field index score output is not implemented yet"); ASSERT_TRUE(result.status().IsNotImplemented()); } @@ -323,7 +328,10 @@ TEST(TableReadTest, TestDependencyReadWithoutCatalogUsesReadOptions) { expected_options[read_via_option] = explicit_header ? "read-view" : "table-view"; ASSERT_EQ(dependency_context->root_path, "warehouse"); ASSERT_EQ(dependency_context->options, expected_options); - ASSERT_EQ(internal_context->GetTableSchema()->Options(), table_options); + // The load merged the read options into the table it built - a table is the product of + // the load - so the schema the read serves carries the explicit header over the one the + // schema was written with, the same value the dependency context sees. + ASSERT_EQ(internal_context->GetTableSchema()->Options(), expected_options); } } From 81d9a5ce908f3e117a66968aa3d3657443b58a27 Mon Sep 17 00:00:00 2001 From: "yonghao.fyh" Date: Fri, 23 Oct 2026 15:43:57 +0800 Subject: [PATCH 11/17] fix --- src/paimon/core/table/source/table_read_test.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/paimon/core/table/source/table_read_test.cpp b/src/paimon/core/table/source/table_read_test.cpp index 072ba3e1d..3cd3b22de 100644 --- a/src/paimon/core/table/source/table_read_test.cpp +++ b/src/paimon/core/table/source/table_read_test.cpp @@ -142,15 +142,15 @@ TEST(TableReadTest, TestFullTextMatchExecutionIsNotImplemented) { ReadContextBuilder builder(table_path); builder.SetReadFieldNames({"f0", "f1"}).SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(auto context, builder.Finish()); - auto result = - table_entry ? table->NewRead(std::move(context)) : TableRead::Create(std::move(context)); + auto result = table_entry ? table->NewRead(std::move(context)) + : TableRead::Create(std::move(context)); ASSERT_NOK_WITH_MSG(result, "FullTextMatch predicate execution is not implemented yet"); ASSERT_TRUE(result.status().IsNotImplemented()); builder.SetReadFieldNames({"f0", "f1", "_f0_INDEX_SCORE"}).SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(context, builder.Finish()); - result = - table_entry ? table->NewRead(std::move(context)) : TableRead::Create(std::move(context)); + result = table_entry ? table->NewRead(std::move(context)) + : TableRead::Create(std::move(context)); ASSERT_NOK_WITH_MSG(result, "Score-producing FullTextMatch cannot appear under OR"); } } @@ -169,8 +169,8 @@ TEST(TableReadTest, TestPerFieldScoreExecutionIsNotImplemented) { ReadContextBuilder builder(path); builder.SetFullTextSearch(search).SetReadFieldNames({"f1", "_f0_INDEX_SCORE"}); ASSERT_OK_AND_ASSIGN(auto context, builder.Finish()); - auto result = - table_entry ? table->NewRead(std::move(context)) : TableRead::Create(std::move(context)); + auto result = table_entry ? table->NewRead(std::move(context)) + : TableRead::Create(std::move(context)); ASSERT_NOK_WITH_MSG(result, "Per-field index score output is not implemented yet"); ASSERT_TRUE(result.status().IsNotImplemented()); } From 231589e04b5c83b7713e6f351359c2ea18689e2b Mon Sep 17 00:00:00 2001 From: "yonghao.fyh" Date: Fri, 23 Oct 2026 17:32:08 +0800 Subject: [PATCH 12/17] fix --- src/paimon/core/table/file_store_table.cpp | 98 ++++++++++++++++++++-- src/paimon/core/table/table_test.cpp | 52 ++++++++++++ 2 files changed, 143 insertions(+), 7 deletions(-) diff --git a/src/paimon/core/table/file_store_table.cpp b/src/paimon/core/table/file_store_table.cpp index c8e50eb62..6a52903b1 100644 --- a/src/paimon/core/table/file_store_table.cpp +++ b/src/paimon/core/table/file_store_table.cpp @@ -19,6 +19,7 @@ #include "paimon/core/table/file_store_table.h" +#include #include #include #include @@ -103,6 +104,26 @@ class MemoryPool; namespace { +/// The identifier a table path names when no catalog name goes with it, the C++ reading of +/// the Java `SchemaManagerUtils#identifierFromPath` over `ignoreIfUnknownDatabase = true`: the +/// directory above the table's names the database when it carries the `.db` suffix, the way a +/// catalog lays its databases out, and a path that names no database - a table sitting directly +/// under the root, say - leaves the database unknown, which is all a caller that only knows the +/// directory can say. +Identifier IdentifierFromPath(const std::string& table_path, const std::string& branch) { + const std::vector paths = StringUtils::Split(table_path, "/"); + if (paths.size() < 2) { + return Identifier(Identifier::kUnknownDatabase, + paths.empty() ? std::string() : paths.front()); + } + const std::string& database = paths[paths.size() - 2]; + const std::string::size_type index = database.rfind(Catalog::DB_SUFFIX); + if (index == std::string::npos) { + return Identifier(Identifier::kUnknownDatabase, paths.back(), branch); + } + return Identifier(database.substr(0, index), paths.back(), branch); +} + /// Recognises a format table behind `table_path`, or returns null when it is not one. /// /// The scan, read, write and commit entry points below all take a path and have to answer the @@ -134,9 +155,10 @@ Result> TryLoadFormatTable( fmt::format("a format table has no branches, so {} cannot be read on branch '{}'", table_path, branch)); } - // A context names a path, never a catalog identifier, so the table is known by the directory - // it sits in. - Identifier identifier(Identifier::kUnknownDatabase, PathUtil::GetName(table_path)); + // A context names a path, never a catalog identifier, so the table is known by the + // directory it sits in - which names the database too, when it carries the `.db` suffix a + // catalog lays its databases out with. + Identifier identifier = IdentifierFromPath(table_path, branch); return FormatTable::Create(file_system, table_path, identifier, checked_pointer_cast(table_schema), /*location_carries_paimon_metadata=*/true, options); @@ -162,6 +184,63 @@ bool FileSystemResolutionMayChange(const TableSchema& schema, return false; } +/// The options a copy is refused to change, mirroring the Java `SchemaManagerUtils# +/// checkAlterTableOption` over the immutable options the C++ side has constants for. These name +/// the physical layout and the file set the data on the table's path was written by: a +/// `bucket-key` the options rename, say, re-hashes the bucket pruning while the files stay +/// hashed by the old key, so the scans a copy serves would filter out the buckets its rows are +/// actually in. The check runs over every option the call gives, before the merge, so it holds +/// on every path that merges options into a table - `Copy` itself and the load a context +/// drives alike. The differences from the Java set: `merge-engine`, `bucket`, +/// `row-tracking.enabled` and `data-evolution.enabled` stay servable as dynamic options here - +/// the C++ tests load tables through context options that move them - and `type`/`branch` are +/// dropped by `Copy` before this point. +const char* const kImmutableOptions[] = { + Options::BUCKET_KEY, + Options::BUCKET_FUNCTION_TYPE, + Options::AGGREGATION_REMOVE_RECORD_ON_DELETE, + Options::PARTIAL_UPDATE_REMOVE_RECORD_ON_DELETE, + Options::PARTIAL_UPDATE_REMOVE_RECORD_ON_SEQUENCE_GROUP, + Options::FORCE_LOOKUP, + Options::INDEX_FILE_IN_DATA_FILE_DIR, + Options::BLOB_FIELD, + Options::FALLBACK_BLOB_DESCRIPTOR_FIELD, + Options::BLOB_VIEW_FIELD, + Options::PK_CLUSTERING_OVERRIDE, +}; + +/// Whether the dynamic options of a copy stay within what the data on the table's path was +/// laid out with - the C++ reading of the Java `AbstractFileStoreTable#checkImmutability`: +/// every option that says what the schema already says moves nothing, an immutable one that +/// says anything else is refused, and an `ignore-delete` carries the directional rule of its +/// own the Java check carries too: it may only ever be turned on, never back off. +Status CheckImmutableOptions(const TableSchema& schema, + const std::map& dynamic_options) { + const std::map& schema_options = schema.Options(); + for (const auto& [key, value] : dynamic_options) { + auto schema_option = schema_options.find(key); + if (schema_option != schema_options.end() && schema_option->second == value) { + continue; + } + if (std::find(std::begin(kImmutableOptions), std::end(kImmutableOptions), key) != + std::end(kImmutableOptions)) { + return Status::Invalid(fmt::format("Change '{}' is not supported yet.", key)); + } + if (key == Options::IGNORE_DELETE) { + // An unset option is one that never ignored deletes, the same default the options + // parse to. + PAIMON_ASSIGN_OR_RAISE(bool old_ignore_delete, + OptionsUtils::GetValueFromMap(schema_options, key, false)); + PAIMON_ASSIGN_OR_RAISE(bool new_ignore_delete, + OptionsUtils::GetValueFromMap(dynamic_options, key)); + if (old_ignore_delete && !new_ignore_delete) { + return Status::Invalid("Cannot change ignore-delete from true to false."); + } + } + } + return Status::OK(); +} + /// The options a context carries, merged into the table the load hands back: a table is the /// product of the load, so the options speak there - the way `Copy` merges them, over the ones /// the schema carries - and `NewXxx` on the product serves them without merging again. A @@ -210,10 +289,11 @@ Result> TableOfLoadedSchema( if (format_table != nullptr) { return std::shared_ptr
(std::move(format_table)); } - // A context names a path, never a catalog identifier, so the table is known by the directory - // it sits in - the same convention `TryLoadFormatTable` follows above. - Identifier table_identifier = - identifier.value_or(Identifier(Identifier::kUnknownDatabase, PathUtil::GetName(path))); + // A context names a path, never a catalog identifier, so the table is known by the + // directory it sits in - which names the database too, when it carries the `.db` suffix a + // catalog lays its databases out with. The branch the schema was read on travels the same + // way, folded into the object name. + Identifier table_identifier = identifier.value_or(IdentifierFromPath(path, branch)); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table, FileStoreTable::Create(explicit_file_system, path, latest_schema, table_identifier, uuid, branch, schema_manager, @@ -540,6 +620,10 @@ Result> FileStoreTable::Copy( std::map overridable = dynamic_options; overridable.erase(Options::TYPE); overridable.erase(Options::BRANCH); + // The physical layout the data on the path was written by is as immovable as the style or + // the branch: the copy serves the same data, so it is refused an option that would lay new + // data out differently than the data it serves was laid out. + PAIMON_RETURN_NOT_OK(CheckImmutableOptions(*loaded_schema, overridable)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr merged_schema, loaded_schema->MergeOptions(overridable)); // The managers are handed over, not rebuilt, when the copy resolves the same file system diff --git a/src/paimon/core/table/table_test.cpp b/src/paimon/core/table/table_test.cpp index b4478918b..3ea6cf54f 100644 --- a/src/paimon/core/table/table_test.cpp +++ b/src/paimon/core/table/table_test.cpp @@ -973,6 +973,58 @@ TEST(TableTest, TestCopyServesTheTableWithTheGivenOptionsOnTop) { ASSERT_FALSE(plan->Splits().empty()); } +TEST(TableTest, TestCopyRefusesThePhysicalOptionsTheDataWasLaidOutWith) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + const Identifier identifier("db", "tbl"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, + CreateManagedTable(dir->GetFileSystem(), dir->Str(), identifier, + /*branch=*/"main", + {{Options::BUCKET, "1"}, {Options::BUCKET_KEY, "id"}})); + + // The data on the path was hashed by the `bucket-key` it was written with: a copy that + // renamed the key would prune by a hash its rows never matched, dropping them silently, so + // the change is refused - at `Copy` and at the load a context drives alike, which merges + // through `Copy` - the way the Java `checkImmutability` refuses it. The same goes for the + // other options that name the physical layout, which a key the schema does not carry + // changes as much as one it carries differently. `merge-engine`, `bucket`, + // `row-tracking.enabled` and `data-evolution.enabled` are the options the Java immutable + // set holds that stay servable here. + ASSERT_NOK_WITH_MSG(table->Copy({{Options::BUCKET_KEY, "v"}}), + "Change 'bucket-key' is not supported yet."); + ASSERT_NOK_WITH_MSG(table->Copy({{Options::BUCKET_FUNCTION_TYPE, "hash"}}), + "Change 'bucket-function.type' is not supported yet."); + ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::shared_ptr
restyled, + table->Copy({{Options::MERGE_ENGINE, "deduplicate"}})); + { + ScanContextBuilder scan_builder(dir->Str()); + scan_builder.AddOption(Options::BUCKET_KEY, "v"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, scan_builder.Finish()); + ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(ctx)), + "Change 'bucket-key' is not supported yet."); + } + + // An option that says what the schema already says moves nothing, and a bucket count is + // the physical option a rescale legitimately rewrites, so a copy may move it. An + // `ignore-delete` may only be turned on, never back off. + ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::shared_ptr
unchanged, + table->Copy({{Options::BUCKET_KEY, "id"}, {Options::BUCKET, "1"}})); + ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::shared_ptr
rescaled, + table->Copy({{Options::BUCKET, "2"}})); + ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::shared_ptr
ignoring_deletes, + table->Copy({{Options::IGNORE_DELETE, "true"}})); + { + ASSERT_OK_AND_ASSIGN(std::shared_ptr
ignore_delete_table, + CreateManagedTable(dir->GetFileSystem(), dir->Str() + "/ignore_delete", + Identifier("db", "igd"), /*branch=*/"main", + {{Options::BUCKET, "1"}, + {Options::BUCKET_KEY, "id"}, + {Options::IGNORE_DELETE, "true"}})); + ASSERT_NOK_WITH_MSG(ignore_delete_table->Copy({{Options::IGNORE_DELETE, "false"}}), + "Cannot change ignore-delete from true to false."); + } +} + TEST(TableTest, TestCreateBuildsTheOperator) { auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); From 043fbb64a7f093a9163bd08766d27c0333643f6b Mon Sep 17 00:00:00 2001 From: "yonghao.fyh" Date: Fri, 23 Oct 2026 17:46:07 +0800 Subject: [PATCH 13/17] update --- src/paimon/core/table/table_test.cpp | 34 ++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/src/paimon/core/table/table_test.cpp b/src/paimon/core/table/table_test.cpp index 3ea6cf54f..b6cc22859 100644 --- a/src/paimon/core/table/table_test.cpp +++ b/src/paimon/core/table/table_test.cpp @@ -1025,6 +1025,40 @@ TEST(TableTest, TestCopyRefusesThePhysicalOptionsTheDataWasLaidOutWith) { } } +TEST(TableTest, TestALoadNamesTheDatabaseThePathLaysOut) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + const std::shared_ptr fs = dir->GetFileSystem(); + + // A context that names no catalog knows the table by its directory: the directory above + // the table's names the database when it carries the `.db` suffix, the way the Java + // `identifierFromPath` reads it, so the table the load serves is named for both - which + // the load of a directory without a schema says in the error it fails with. The option + // the context carries is what drives the load to that error: it hands the table the merge + // would build, which is where the name is spoken. + const std::string table_dir = dir->Str() + "/my_db.db/my_table"; + ASSERT_OK(fs->Mkdirs(table_dir)); + { + ScanContextBuilder builder(table_dir); + builder.AddOption("read.batch-size", "2"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, builder.Finish()); + ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(ctx)), + "not found latest schema of my_db.my_table"); + } + + // A table sitting directly under the root, with no `.db` directory above it, names no + // database: the table is known by its directory alone, the way it was before. + const std::string loose_table_dir = dir->Str() + "/loose_table"; + ASSERT_OK(fs->Mkdirs(loose_table_dir)); + { + ScanContextBuilder builder(loose_table_dir); + builder.AddOption("read.batch-size", "2"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, builder.Finish()); + ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(ctx)), + "not found latest schema of loose_table"); + } +} + TEST(TableTest, TestCreateBuildsTheOperator) { auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); From cc12dd9afe00ef2d781f2e58558f5e100b36521e Mon Sep 17 00:00:00 2001 From: "yonghao.fyh" Date: Mon, 26 Oct 2026 18:31:39 +0800 Subject: [PATCH 14/17] fix --- include/paimon/catalog/catalog.h | 14 +- include/paimon/catalog/table.h | 48 +- include/paimon/commit_context.h | 10 +- include/paimon/read_context.h | 10 +- include/paimon/scan_context.h | 13 +- include/paimon/table/format/format_table.h | 46 +- include/paimon/write_context.h | 10 +- src/paimon/CMakeLists.txt | 3 + src/paimon/core/catalog/catalog_utils.cpp | 27 + src/paimon/core/catalog/catalog_utils.h | 17 +- .../core/catalog/file_system_catalog.cpp | 15 +- src/paimon/core/catalog/snapshot_commit.h | 27 + ...eted_primary_key_index_maintainer_test.cpp | 26 +- .../append_only_file_store_write_test.cpp | 22 +- .../core/operation/internal_read_context.cpp | 48 + .../core/operation/internal_read_context.h | 31 +- .../core/operation/orphan_files_cleaner.cpp | 11 +- src/paimon/core/schema/schema_validation.cpp | 49 + src/paimon/core/schema/schema_validation.h | 9 + .../table/append_only_file_store_table.cpp | 16 +- .../core/table/append_only_file_store_table.h | 6 - src/paimon/core/table/file_store.cpp | 49 + src/paimon/core/table/file_store.h | 154 ++-- src/paimon/core/table/file_store_table.cpp | 857 +++--------------- src/paimon/core/table/file_store_table.h | 123 ++- src/paimon/core/table/format/format_table.cpp | 41 +- .../table/primary_key_file_store_table.cpp | 17 +- .../core/table/primary_key_file_store_table.h | 6 - .../core/table/source/realtime_table_scan.cpp | 47 + .../core/table/source/realtime_table_scan.h | 9 + .../table/system/audit_log_system_table.cpp | 14 +- .../core/table/system/binlog_system_table.cpp | 3 + .../core/table/system/global_system_tables.h | 9 +- .../table/system/in_memory_system_table.cpp | 5 + .../system/read_optimized_system_table.cpp | 14 +- src/paimon/core/table/system/system_table.cpp | 9 + src/paimon/core/table/system/system_table.h | 28 +- .../core/table/system/system_table_test.cpp | 59 ++ src/paimon/core/table/table.cpp | 33 + src/paimon/core/table/table_test.cpp | 315 ++++++- src/paimon/rest/rest_catalog.cpp | 31 +- src/paimon/rest/rest_catalog.h | 4 +- src/paimon/rest/rest_catalog_test.cpp | 41 + test/inte/realtime_write_inte_test.cpp | 57 +- test/inte/scan_and_read_inte_test.cpp | 79 +- test/inte/scan_inte_test.cpp | 27 +- .../write_and_read_inte_with_index_test.cpp | 3 +- 47 files changed, 1146 insertions(+), 1346 deletions(-) create mode 100644 src/paimon/core/table/table.cpp diff --git a/include/paimon/catalog/catalog.h b/include/paimon/catalog/catalog.h index 0bb578383..033e7a822 100644 --- a/include/paimon/catalog/catalog.h +++ b/include/paimon/catalog/catalog.h @@ -147,12 +147,14 @@ class PAIMON_EXPORT Catalog { /// Gets a table. /// - /// The table's `type` option decides what is handed back: a `format-table` comes back as - /// the `FormatTable` it is - a directory of data files, with no snapshot and no manifest - - /// and every other implemented type as the managed table it is. A `T$system` identifier - /// names a read-only view instead, which comes back as the `SystemTable` it is. All go - /// through this one `Table` interface: `NewScan()` and `NewRead()` serve any of them, - /// `NewWrite()` and `NewCommit()` the writable ones. + /// The table's `type` option decides what is handed back: a `format-table` comes back as the + /// `FormatTable` it is - a directory of data files, with no snapshot and no manifest - and + /// every other implemented type as the managed table it is, while a `T$system` identifier names + /// a read-only `SystemTable`. All go through this one `Table` interface: `NewScan()` and + /// `NewRead()` serve any of them, `NewWrite()` and `NewCommit()` the writable ones. + /// + /// A global system table of the "sys" database reads its rows lazily through the catalog it + /// came from, so a catalog that serves one must be owned by a `std::shared_ptr`. /// /// @param identifier Identifier of the table to get. /// @return A result containing the table, or an error status. diff --git a/include/paimon/catalog/table.h b/include/paimon/catalog/table.h index a31d3100d..3364f6eff 100644 --- a/include/paimon/catalog/table.h +++ b/include/paimon/catalog/table.h @@ -44,36 +44,33 @@ class WriteContext; /// A table provides basic abstraction for table type. /// -/// The state a table of every kind carries - file system, root path, schema and identifier - lives -/// here, and each kind of table fills the operations in: the operators below create this table's -/// scan, read, write or commit, from a context the caller finishes through its builder. What the -/// context asks for that this table cannot honour is refused by name by the implementation rather +/// The state every kind of table carries - file system, root path, schema and identifier - lives +/// here, and each kind fills the operations in. The `NewXxx` operators below are the table-first +/// entry points: `TableScan::Create()` and friends load the table a context names and hand the +/// context to them. What a context asks for that the table cannot honour is refused by name rather /// than silently dropped. class PAIMON_EXPORT Table { public: virtual ~Table() = default; - /// Returns the file system the table was opened with, which serves the table's - /// files. A catalog that hands out per-table file systems opens a table with the - /// one for that table; it is null when the opener had none. The `NewXxx` - /// operations read the table through it. + /// Returns the file system the table was opened with, null when the opener had none. A + /// catalog that hands out per-table file systems opens a table with the one for that table. std::shared_ptr GetFileSystem() const { return file_system_; } - /// Returns the root path of the table in its file system. It is empty when the - /// opener had no location for the table, which a mock in a test is one case of. - /// The `NewXxx` operations serve the table from it. + /// Returns the root path of the table in its file system. It is empty when the opener had no + /// location for the table, which a mock in a test is one case of. const std::string& GetRootPath() const { return root_path_; } - /// A name to identify this table, which each kind of table answers for itself: a managed or - /// format table is named by its identifier, a system table by its own kind. + /// A name to identify this table: a managed or format table answers its identifier's object + /// name, a system table its own kind. virtual std::string Name() const = 0; - /// Full name of the table, which each kind of table answers for itself. A managed or format - /// table answers database.tableName, a system table the same name as `Name()`. + /// Full name of the table: `database.tableName` for a managed or format table, the same name + /// as `Name()` for a system table. virtual std::string FullName() const = 0; /// Returns the metastore UUID, falling back to `FullName()`. @@ -93,41 +90,34 @@ class PAIMON_EXPORT Table { return schema_; } - /// Copies this table with `dynamic_options` on top of the options it carries, which is - /// the precedence every context builder promises for a table it was handed rather than loaded - /// itself: an option given at the call wins over the one the schema stored. - /// - /// `type` is not overridable: it is structural and is read from the schema alone. + /// Copies this table with `dynamic_options` on top of the options it carries: an option given + /// at the call wins over the one the schema stored, the precedence every context builder + /// promises. `type` is not overridable - it is structural and read from the schema alone. /// /// @param dynamic_options Options given at the call. /// @return A result containing the copied table, or the error copying it reports. virtual Result> Copy( const std::map& dynamic_options) const = 0; - /// Creates a scan of this table from a finished scan context, the table-first entry point - /// that `TableScan::Create()` loads the table the context names and hands the context to. + /// Creates a scan of this table from a finished scan context. /// @param context The finished scan context. /// @return The scan, or the error building it reports. virtual Result> NewScan( std::unique_ptr context) const = 0; - /// Creates a read of this table from a finished read context, the table-first entry point - /// that `TableRead::Create()` loads the table the context names and hands the context to. + /// Creates a read of this table from a finished read context. /// @param context The finished read context. /// @return The read, or the error building it reports. virtual Result> NewRead( std::unique_ptr context) const = 0; - /// Creates a write of this table from a finished write context, the table-first entry point - /// that `FileStoreWrite::Create()` loads the table the context names and hands the context to. + /// Creates a write of this table from a finished write context. /// @param context The finished write context. /// @return The write, or the error building it reports. virtual Result> NewWrite( std::unique_ptr context) const = 0; - /// Creates a commit of this table from a finished commit context, the table-first entry point - /// that `FileStoreCommit::Create()` loads the table the context names and hands the context - /// to. + /// Creates a commit of this table from a finished commit context. /// @param context The finished commit context. /// @return The commit, or the error building it reports. virtual Result> NewCommit( diff --git a/include/paimon/commit_context.h b/include/paimon/commit_context.h index a70cacb7a..556db7bd2 100644 --- a/include/paimon/commit_context.h +++ b/include/paimon/commit_context.h @@ -156,12 +156,10 @@ class PAIMON_EXPORT CommitContextBuilder { ~CommitContextBuilder(); - /// Set a configuration options map. The options take part in loading the table a commit - /// serves: the load merges them over the ones the table's schema carries and the table it - /// hands back serves the merged ones. A caller holding a table already loaded - a load - /// that is over - changes it through `Table::Copy()`; an option contradicting a table held - /// that way is refused when a commit is asked of it. A format table takes the options as - /// its own. + /// Set a configuration options map to set some option entries which are not defined in the + /// table schema or whose values you want to overwrite. A load merges them into the table it + /// hands back; an option contradicting a table you loaded already is refused, so change that + /// one through `Table::Copy()`. /// @note The options map will clear the options added by `AddOption()` before. /// @param options The configuration options map. /// @return Reference to this builder for method chaining. diff --git a/include/paimon/read_context.h b/include/paimon/read_context.h index 8cf565996..d01e28df5 100644 --- a/include/paimon/read_context.h +++ b/include/paimon/read_context.h @@ -324,12 +324,10 @@ class PAIMON_EXPORT ReadContextBuilder { /// When set, read_field_ids and read_field_names are ignored. ReadContextBuilder& SetReadSchema(std::unique_ptr read_schema); - /// Set a configuration options map. The options take part in loading the table a read - /// serves: the load merges them over the ones the table's schema carries and the table it - /// hands back serves the merged ones. A caller holding a table already loaded - a load - /// that is over - changes it through `Table::Copy()`; an option contradicting a table held - /// that way is refused when a read is asked of it. A format table takes the options as its - /// own. + /// Set a configuration options map to set some option entries which are not defined in the + /// table schema or whose values you want to overwrite. A load merges them into the table it + /// hands back; an option contradicting a table you loaded already is refused, so change that + /// one through `Table::Copy()`. /// @note The options map will clear the options added by `AddOption()` before. /// @param options The configuration options map. /// @return Reference to this builder for method chaining. diff --git a/include/paimon/scan_context.h b/include/paimon/scan_context.h index fe57afb58..2e3f8d9a8 100644 --- a/include/paimon/scan_context.h +++ b/include/paimon/scan_context.h @@ -208,15 +208,12 @@ class PAIMON_EXPORT ScanContextBuilder { ScanContextBuilder& WithRealtimeContext( const std::shared_ptr& realtime_context); - /// Add a single configuration option. See `SetOptions()` for how the options a context - /// carries are served. + /// Add a single configuration option, served the way `SetOptions()` describes. ScanContextBuilder& AddOption(const std::string& key, const std::string& value); - /// Set a configuration options map. The options take part in loading the table a scan - /// serves: the load merges them over the ones the table's schema carries and the table it - /// hands back serves the merged ones. A caller holding a table already loaded - a load - /// that is over - changes it through `Table::Copy()`; an option contradicting a table held - /// that way is refused when a scan is asked of it. A format table takes the options as its - /// own. + /// Set a configuration options map to set some option entries which are not defined in the + /// table schema or whose values you want to overwrite. A load merges them into the table it + /// hands back; an option contradicting a table you loaded already is refused, so change that + /// one through `Table::Copy()`. /// @note The options map will clear the options added by `AddOption()` before. /// @param options The configuration options map. /// @return Reference to this builder for method chaining. diff --git a/include/paimon/table/format/format_table.h b/include/paimon/table/format/format_table.h index 88444d360..c5bfe60d3 100644 --- a/include/paimon/table/format/format_table.h +++ b/include/paimon/table/format/format_table.h @@ -48,12 +48,11 @@ class FileSystem; /// Writes only insert - there is nowhere to record an update or a delete - so a read fills /// `_VALUE_KIND` with inserts throughout. /// -/// A format table is a `Table`, so the operators it inherits create its scan, read, write and -/// commit from this table, with the context's options applied on top of the ones it carries. The -/// entry points every other table uses remain open too - `TableScan::Create()` and friends - each -/// of which recognises a format table from the schema under the table path, or takes one already -/// loaded through the `FormatTable` constructor every context builder has, which is how a table -/// whose schema lives in a metastore is reached. +/// A format table is a `Table`, so it is reached both ways: through the `NewXxx` operators below, +/// and through the entry points every other table uses - `TableScan::Create()` and friends - which +/// recognise one from the schema under the table path, or take one already loaded through the +/// `FormatTable` constructor every context builder has, how a table whose schema lives in a +/// metastore is reached. /// /// Every instance lives in a `shared_ptr`: a copy that adds nothing hands the same one back. /// @@ -117,8 +116,7 @@ class PAIMON_EXPORT FormatTable : public Table, public std::enable_shared_from_t return identifier_.GetFullName(); } - /// Copies the table with `dynamic_options` on top of the options it carries, the precedence - /// every context builder promises. The same table comes back when there is nothing to add. + /// The same table comes back when there is nothing to add. Result> Copy( const std::map& dynamic_options) const override; @@ -160,9 +158,8 @@ class PAIMON_EXPORT FormatTable : public Table, public std::enable_shared_from_t return options_; } - /// Schema of the table, including its partition fields. It hides `Table::LatestSchema()`, - /// which returns the same schema as the base every table sees; this one hands it back as the - /// `DataSchema` a format table is always created with. + /// Schema of the table, including its partition fields. Hides `Table::LatestSchema()` to hand + /// back the same schema as the `DataSchema` a format table is always created with. std::shared_ptr LatestSchema() const { return std::static_pointer_cast(schema_); } @@ -179,25 +176,16 @@ class PAIMON_EXPORT FormatTable : public Table, public std::enable_shared_from_t return location_carries_paimon_metadata_; } - /// Creates a scan of this table from a finished scan context, refusing by name what a format - /// table cannot honour. The context's options are applied on top of the ones this table - /// carries, the precedence every context builder promises. + /// The four operators below create this table's scan, read, write and commit, applying the + /// context's options on top of the ones this table carries and refusing by name what a + /// format table cannot honour. Result> NewScan(std::unique_ptr context) const override; - /// Creates a read of this table from a finished read context, refusing by name what a format - /// table cannot honour. The context's options are applied on top of the ones this table - /// carries, the precedence every context builder promises. Result> NewRead(std::unique_ptr context) const override; - /// Creates a write of this table from a finished write context, refusing by name what a format - /// table cannot honour. The context's options are applied on top of the ones this table - /// carries, the precedence every context builder promises. Result> NewWrite( std::unique_ptr context) const override; - /// Creates a commit of this table from a finished commit context, refusing by name what a - /// format table cannot honour. The context's options are applied on top of the ones this - /// table carries, the precedence every context builder promises. Result> NewCommit( std::unique_ptr context) const override; @@ -208,18 +196,14 @@ class PAIMON_EXPORT FormatTable : public Table, public std::enable_shared_from_t const std::string& file_compression, const std::string& partition_default_name, bool partition_only_value_in_path, bool location_carries_paimon_metadata); - /// This table as a `shared_ptr`, which a copy that adds nothing hands back. - /// - /// `shared_from_this()` on a const method hands back a `shared_ptr`, while - /// the copy below needs a mutable one; the cast is safe because the table is never written to. + /// This table as a `shared_ptr`. `shared_from_this()` on a const method hands back a + /// `shared_ptr`, while the copy below needs a mutable one; the cast is + /// safe because the table is never written to. std::shared_ptr SharedSelf() const { return std::const_pointer_cast(shared_from_this()); } - /// Copies this table with `dynamic_options` on top of the options it carries, as a format - /// table rather than the `Table` `Copy()` hands back, which is what its operators need. The - /// same table comes back when there is nothing to add; `type` is not overridable, and - /// `Create()` keeps the schema's own. + /// `Copy()` as a `FormatTable` rather than a `Table`, which is what its operators need. Result> CopySelf( const std::map& dynamic_options) const; diff --git a/include/paimon/write_context.h b/include/paimon/write_context.h index 23195793f..f1d3ac06e 100644 --- a/include/paimon/write_context.h +++ b/include/paimon/write_context.h @@ -183,12 +183,10 @@ class PAIMON_EXPORT WriteContextBuilder { ~WriteContextBuilder(); - /// Set a configuration options map. The options take part in loading the table a write - /// serves: the load merges them over the ones the table's schema carries and the table it - /// hands back serves the merged ones. A caller holding a table already loaded - a load - /// that is over - changes it through `Table::Copy()`; an option contradicting a table held - /// that way is refused when a write is asked of it. A format table takes the options as its - /// own. + /// Set a configuration options map to set some option entries which are not defined in the + /// table schema or whose values you want to overwrite. A load merges them into the table it + /// hands back; an option contradicting a table you loaded already is refused, so change that + /// one through `Table::Copy()`. /// @note The options map will clear the options added by `AddOption()` before. /// @param options The configuration options map. /// @return Reference to this builder for method chaining. diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt index 8d1234c4d..87dffa570 100644 --- a/src/paimon/CMakeLists.txt +++ b/src/paimon/CMakeLists.txt @@ -267,6 +267,7 @@ set(PAIMON_CORE_SRCS core/catalog/catalog_utils.cpp core/catalog/file_system_catalog.cpp core/catalog/identifier.cpp + core/catalog/snapshot_commit.cpp core/core_options.cpp core/deletionvectors/deletion_vector.cpp core/deletionvectors/deletion_file_writer.cpp @@ -447,6 +448,7 @@ set(PAIMON_CORE_SRCS core/table/bucket_mode.cpp core/table/file_store.cpp core/table/file_store_table.cpp + core/table/file_store_table_factory.cpp core/table/format/format_file_listing.cpp core/table/format/format_file_naming.cpp core/table/format/format_path_validation.cpp @@ -492,6 +494,7 @@ set(PAIMON_CORE_SRCS core/table/system/system_table.cpp core/table/system/system_table_scan.cpp core/table/system/system_table_schema.cpp + core/table/table.cpp core/tag/tag.cpp core/utils/branch_manager.cpp core/utils/blob_view_lookup.cpp diff --git a/src/paimon/core/catalog/catalog_utils.cpp b/src/paimon/core/catalog/catalog_utils.cpp index 721942a88..4cf128a94 100644 --- a/src/paimon/core/catalog/catalog_utils.cpp +++ b/src/paimon/core/catalog/catalog_utils.cpp @@ -22,10 +22,12 @@ #include #include #include +#include #include "fmt/format.h" #include "paimon/catalog/catalog.h" #include "paimon/common/utils/path_util.h" +#include "paimon/common/utils/string_utils.h" #include "paimon/core/options/table_type.h" #include "paimon/core/utils/branch_manager.h" #include "paimon/defs.h" @@ -43,6 +45,21 @@ Status SystemTableError(const Identifier& identifier, const std::string& action) } // namespace +Identifier CatalogUtils::IdentifierFromPath(const std::string& table_path, + const std::string& branch) { + const std::vector paths = StringUtils::Split(table_path, "/"); + if (paths.size() < 2) { + return Identifier(Identifier::kUnknownDatabase, + paths.empty() ? std::string() : paths.front()); + } + const std::string& database = paths[paths.size() - 2]; + const std::string::size_type index = database.rfind(Catalog::DB_SUFFIX); + if (index == std::string::npos) { + return Identifier(Identifier::kUnknownDatabase, paths.back(), branch); + } + return Identifier(database.substr(0, index), paths.back(), branch); +} + bool CatalogUtils::IsSystemDatabase(const std::string& db_name) { return db_name == Catalog::SYSTEM_DATABASE_NAME; } @@ -74,6 +91,16 @@ Result CatalogUtils::BranchIdentifier(const Identifier& identifier, return Identifier(identifier.GetDatabaseName(), data_table_name, branch); } +Result> CatalogUtils::BranchIdentifier( + const std::optional& identifier, const std::string& branch) { + if (!identifier) { + return std::optional(); + } + PAIMON_ASSIGN_OR_RAISE(Identifier branch_identifier, + BranchIdentifier(identifier.value(), branch)); + return std::optional(std::move(branch_identifier)); +} + Status CatalogUtils::CheckNotBranch(const Identifier& identifier, const std::string& action) { PAIMON_ASSIGN_OR_RAISE(std::optional branch, identifier.GetBranchName()); if (branch) { diff --git a/src/paimon/core/catalog/catalog_utils.h b/src/paimon/core/catalog/catalog_utils.h index 23cb77203..58775d4df 100644 --- a/src/paimon/core/catalog/catalog_utils.h +++ b/src/paimon/core/catalog/catalog_utils.h @@ -19,6 +19,7 @@ #pragma once #include +#include #include #include "paimon/catalog/identifier.h" @@ -31,13 +32,19 @@ class Catalog; class FormatTable; class Schema; -/// Checks shared by the catalog implementations. Every check takes an `action` naming -/// the rejected operation in the error message, e.g. "dropTable". +/// The identifier tools and the checks shared by the catalog implementations. Every check takes an +/// `action` naming the rejected operation in the error message, e.g. "dropTable". class CatalogUtils { public: CatalogUtils() = delete; ~CatalogUtils() = delete; + /// The identifier a table path names when no catalog name goes with it, the C++ reading of the + /// Java `SchemaManagerUtils#identifierFromPath` over `ignoreIfUnknownDatabase = true`: the + /// directory above the table's names the database when it carries the `.db` suffix a catalog + /// lays its databases out with, and a path that names no database leaves it unknown. + static Identifier IdentifierFromPath(const std::string& table_path, const std::string& branch); + /// Returns whether `db_name` is the reserved system database "sys". static bool IsSystemDatabase(const std::string& db_name); @@ -58,6 +65,12 @@ class CatalogUtils { static Result BranchIdentifier(const Identifier& identifier, const std::string& branch); + /// The identifier a catalog knows the table by, on `branch`, when there is one to fold a + /// branch into: a caller without an identifier has none to ask the catalog about, and gets + /// none back rather than an error. + static Result> BranchIdentifier( + const std::optional& identifier, const std::string& branch); + /// Fails when `db_name` cannot be used as a single path component, which is required to /// keep the database path under the warehouse. static Status CheckValidDatabaseName(const std::string& db_name); diff --git a/src/paimon/core/catalog/file_system_catalog.cpp b/src/paimon/core/catalog/file_system_catalog.cpp index d3c6bd92e..fdcddf20e 100644 --- a/src/paimon/core/catalog/file_system_catalog.cpp +++ b/src/paimon/core/catalog/file_system_catalog.cpp @@ -287,6 +287,9 @@ Result> FileSystemCatalog::LoadTableSchema( if (!supported) { return Status::NotExist(fmt::format("{} not exist", identifier.ToString())); } + // The table built here is transient - its schema is read within this call and nothing + // outlives it - so its context keeps no shared ownership of the catalog. A catalog that + // hands a global system table back out (`RestCatalog::GetTable`) retains it instead. GlobalSystemTableContext context; context.catalog = this; context.fs = fs_; @@ -316,8 +319,7 @@ Result> FileSystemCatalog::LoadTableSchema( Result> FileSystemCatalog::GetTable(const Identifier& identifier) const { PAIMON_ASSIGN_OR_RAISE(bool is_system_table, identifier.IsSystemTable()); if (is_system_table) { - // A system table reads the files of the data table it belongs to, through the file - // system of that table's catalog, and the schema it exposes is its own read-only one. + // A system table reads the data table's files through this catalog's file system. return LoadSystemTable(identifier); } // The schema is read on the table's own branch: `root/branch/branch-` below the @@ -330,8 +332,7 @@ Result> FileSystemCatalog::GetTable(const Identifier& ide } PAIMON_ASSIGN_OR_RAISE(std::string table_path, GetTableLocation(identifier)); // The schema's `type` says what the table is. A format table is the same directory read as - // plain data, so the schema read above is all it takes to serve one: no manifest is looked - // for, and the checks a format table adds run in `FormatTable::Create`. + // plain data, so the schema read above is all it takes to serve one; no manifest is read. std::shared_ptr data_schema = checked_pointer_cast(*latest_schema); PAIMON_ASSIGN_OR_RAISE(bool is_format_table, TableTypeDefine::IsFormatTable(data_schema->Options())); @@ -347,8 +348,7 @@ Result> FileSystemCatalog::GetTable(const Identifier& ide // table rather than at the first read. PAIMON_RETURN_NOT_OK(CatalogUtils::CheckManagedTableType(identifier, *latest_schema, "FileSystemCatalog::GetTable")); - // The branch the schema above was read on, which the table carries so its managers read - // the same branch: `GetBranchNameOrDefault` folds any spelling of `main`. + // The branch the schema above was read on, so the table's managers read the same one. PAIMON_ASSIGN_OR_RAISE(std::string branch, identifier.GetBranchNameOrDefault()); return FileStoreTable::Create(fs_, table_path, checked_pointer_cast(*latest_schema), identifier, /*uuid=*/"", branch); @@ -374,8 +374,7 @@ Result> FileSystemCatalog::LoadSystemTable( if (branch) { dynamic_options[Options::BRANCH] = branch.value(); } - // A system table reads the files of the data table it belongs to, so its root path is - // that table's. + // A system table reads the files of the data table it belongs to, so its root path is that one. PAIMON_ASSIGN_OR_RAISE(std::string table_path, GetTableLocation(data_identifier)); return SystemTableLoader::Load(system_table_name.value(), fs_, table_path, latest_schema.value(), dynamic_options); diff --git a/src/paimon/core/catalog/snapshot_commit.h b/src/paimon/core/catalog/snapshot_commit.h index 9500205cb..fa451c204 100644 --- a/src/paimon/core/catalog/snapshot_commit.h +++ b/src/paimon/core/catalog/snapshot_commit.h @@ -18,20 +18,47 @@ #pragma once +#include #include #include #include +#include "paimon/catalog/identifier.h" #include "paimon/core/partition/partition_statistics.h" #include "paimon/core/snapshot.h" namespace paimon { +class Catalog; +class FileSystem; +class SnapshotManager; + /// Interface to commit snapshot atomically. class SnapshotCommit { public: virtual ~SnapshotCommit() = default; + /// The commit a table publishes its snapshots through, the C++ reading of the Java + /// `CatalogEnvironment#snapshotCommit`: a catalog that version-manages the table commits + /// through it, a caller that only wants the request built gets one with no catalog attached, + /// and everything else publishes by renaming the snapshot file into place. + /// + /// @param catalog The catalog the commit was handed, or null for one that has none. + /// @param catalog_identifier The identifier the catalog knows the table by, on the branch the + /// commit commits to. A version-managing catalog needs one and is refused without it. + /// @param table_id The catalog's UUID for the table, captured before the changes were + /// prepared; null when there is none. + /// @param use_rest_catalog_commit Whether the caller asks for the request rather than for the + /// catalog to send it. + /// @param file_system The file system the snapshot file is renamed through. + /// @param snapshot_manager The manager that names the snapshot file renamed. + static Result> Create( + const std::shared_ptr& catalog, + const std::optional& catalog_identifier, + const std::optional& table_id, bool use_rest_catalog_commit, + const std::shared_ptr& file_system, + const std::shared_ptr& snapshot_manager); + /// @param base_snapshot_uuid Base snapshot UUID; null for an absent or legacy snapshot. /// @param snapshot Snapshot to be committed. /// @param branch Branch to publish the snapshot on; an empty name is the main branch. diff --git a/src/paimon/core/index/pk/bucketed_primary_key_index_maintainer_test.cpp b/src/paimon/core/index/pk/bucketed_primary_key_index_maintainer_test.cpp index 769bbf326..81803ed90 100644 --- a/src/paimon/core/index/pk/bucketed_primary_key_index_maintainer_test.cpp +++ b/src/paimon/core/index/pk/bucketed_primary_key_index_maintainer_test.cpp @@ -33,7 +33,6 @@ #include "gtest/gtest.h" #include "paimon/catalog/catalog.h" #include "paimon/catalog/identifier.h" -#include "paimon/catalog/table.h" #include "paimon/commit_context.h" #include "paimon/common/utils/path_util.h" #include "paimon/core/global_index/indexed_split_impl.h" @@ -474,21 +473,15 @@ TEST_P(BucketedPrimaryKeyIndexMaintainerTest, BuildsRestoresAndReplacesCompacted ASSERT_EQ(0, data_split_count); ReadContextBuilder read_builder(table_path_); + // The source build above keeps lazy decoding enabled. Disable it only for the final data-row + // assertion, whose predicate reader is outside the maintenance path covered by this test. read_builder.SetReadFieldNames({"id", "value"}) .SetPredicate(value_predicate) + .AddOption("orc.read.enable-lazy-decoding", "false") .EnablePredicateFilter(true); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); - // The source build above keeps lazy decoding enabled. Disable it only for the final data-row - // assertion, whose predicate reader is outside the maintenance path covered by this test. - // The table is copied with it off because a read through the loaded table refuses an option - // that contradicts the one it carries. - ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, Catalog::Create(directory_->Str(), {})); - ASSERT_OK_AND_ASSIGN(std::shared_ptr
loaded_table, - catalog->GetTable(Identifier("db", "table"))); - ASSERT_OK_AND_ASSIGN(std::shared_ptr
copied_table, - loaded_table->Copy({{"orc.read.enable-lazy-decoding", "false"}})); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - copied_table->NewRead(std::move(read_context))); + TableRead::Create(std::move(read_context))); ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_reader, table_read->CreateReader(plan->Splits())); ASSERT_OK_AND_ASSIGN(std::shared_ptr result, @@ -714,18 +707,11 @@ TEST_P(BucketedPrimaryKeyIndexMaintainerTest, ContinuesCompactionWhenIndexBuildF ReadContextBuilder read_builder(table_path_); read_builder.SetReadFieldNames({"id", "value"}) .SetPredicate(value_predicate) + .AddOption("orc.read.enable-lazy-decoding", "false") .EnablePredicateFilter(true); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); - // Like the test above, the assertion reads with lazy decoding disabled, so it reads through - // a copy of the table carrying it off: a read through the loaded table refuses an option that - // contradicts the one it carries. - ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, Catalog::Create(directory_->Str(), {})); - ASSERT_OK_AND_ASSIGN(std::shared_ptr
loaded_table, - catalog->GetTable(Identifier("db", "table"))); - ASSERT_OK_AND_ASSIGN(std::shared_ptr
copied_table, - loaded_table->Copy({{"orc.read.enable-lazy-decoding", "false"}})); ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, - copied_table->NewRead(std::move(read_context))); + TableRead::Create(std::move(read_context))); ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_reader, table_read->CreateReader(plan->Splits())); ASSERT_OK_AND_ASSIGN(std::shared_ptr result, diff --git a/src/paimon/core/operation/append_only_file_store_write_test.cpp b/src/paimon/core/operation/append_only_file_store_write_test.cpp index 3ce48ee2c..bfeaefced 100644 --- a/src/paimon/core/operation/append_only_file_store_write_test.cpp +++ b/src/paimon/core/operation/append_only_file_store_write_test.cpp @@ -36,7 +36,6 @@ #include "gtest/gtest.h" #include "paimon/catalog/catalog.h" #include "paimon/catalog/identifier.h" -#include "paimon/catalog/table.h" #include "paimon/commit_context.h" #include "paimon/common/data/binary_row.h" #include "paimon/common/data/binary_row_writer.h" @@ -514,25 +513,10 @@ TEST_F(AppendOnlyFileStoreWriteTest, TestSharedShreddingNewWriterIgnoresExisting /*commit_identifier=*/0); Commit(table_path, avro_options, avro_commit_msgs); - // The second batch is written through a copy of the table carrying the shredding options: - // an option on the context would contradict the table the catalog loaded, which a write - // through that table refuses. - ASSERT_OK_AND_ASSIGN(auto catalog, Catalog::Create(dir->Str(), avro_options)); - ASSERT_OK_AND_ASSIGN(std::shared_ptr
avro_table, - catalog->GetTable(Identifier("foo", "bar"))); - ASSERT_OK_AND_ASSIGN(std::shared_ptr
shredded_table, - avro_table->Copy(shredding_options)); - WriteContextBuilder shredded_builder(table_path, commit_user_); - ASSERT_OK_AND_ASSIGN(std::unique_ptr shredded_context, shredded_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto shredded_write, - shredded_table->NewWrite(std::move(shredded_context))); - ASSERT_OK(shredded_write->Write(MakeBatch(logical_schema, R"([ + auto second_commit_msgs = WriteAndPrepare(table_path, logical_schema, shredding_options, R"([ [2, [["c", 3], ["d", 4], ["e", 5]]] - ])"))); - ASSERT_OK_AND_ASSIGN(auto second_commit_msgs, - shredded_write->PrepareCommit(/*wait_compaction=*/false, - /*commit_identifier=*/1)); - ASSERT_OK(shredded_write->Close()); + ])", + /*commit_identifier=*/1); auto second_file_schema = ReadDataFileSchema(table_path, OnlyNewFile(second_commit_msgs), shredding_options); auto second_meta = ShreddingMeta(second_file_schema, /*field_index=*/1); diff --git a/src/paimon/core/operation/internal_read_context.cpp b/src/paimon/core/operation/internal_read_context.cpp index 3eb169ac7..8866eae9f 100644 --- a/src/paimon/core/operation/internal_read_context.cpp +++ b/src/paimon/core/operation/internal_read_context.cpp @@ -19,6 +19,7 @@ #include "paimon/core/operation/internal_read_context.h" #include +#include #include #include #include @@ -44,7 +45,10 @@ #include "paimon/common/utils/url_utils.h" #include "paimon/core/options/map_storage_layout.h" #include "paimon/core/schema/arrow_schema_validator.h" +#include "paimon/core/schema/schema_manager.h" +#include "paimon/core/utils/branch_manager.h" #include "paimon/core/utils/nested_projection_utils.h" +#include "paimon/defs.h" #include "paimon/predicate/compound_predicate.h" #include "paimon/predicate/full_text_search.h" #include "paimon/predicate/predicate_utils.h" @@ -296,6 +300,50 @@ Result> InternalReadContext::Create( return CreateFromCoreOptions(context, table_schema, std::move(core_options), path); } +Result> InternalReadContext::CreateForFallbackBranch( + const std::shared_ptr& context, const std::string& branch, + const std::shared_ptr& file_system, const std::string& root_path, + const std::map& table_dynamic_options) { + std::shared_ptr table_schema; + const auto& specific_table_schema = context->GetSpecificTableSchema(); + if (branch == BranchManager::DEFAULT_MAIN_BRANCH && specific_table_schema) { + PAIMON_ASSIGN_OR_RAISE(table_schema, + TableSchema::CreateFromJson(specific_table_schema.value())); + } else { + auto schema_manager = std::make_shared(file_system, root_path, branch); + PAIMON_ASSIGN_OR_RAISE(std::optional> latest_schema, + schema_manager->Latest()); + if (!latest_schema) { + return Status::Invalid( + fmt::format("schema file not found in path {}, branch {}", root_path, branch)); + } + table_schema = latest_schema.value(); + } + assert(table_schema); + auto options = table_schema->Options(); + // The dynamic options the load merged into this table - and the ones its copies accumulated - + // speak for the read, and the fallback branch serves the same read: they are merged over the + // fallback's own schema options the way the load merged them over the table's, so the fallback + // reader recognises what the main one does. `type` stays the branch's own: neither a copy nor + // a fallback restyles the table. + for (const auto& [key, value] : table_dynamic_options) { + if (key != Options::TYPE) { + options[key] = value; + } + } + for (const auto& [key, value] : context->GetOptions()) { + if (key != Options::TYPE) { + options[key] = value; + } + } + if (branch != BranchManager::DEFAULT_MAIN_BRANCH) { + options[Options::BRANCH] = branch; + } else { + options.erase(Options::BRANCH); + } + return Create(context, table_schema, options, file_system, &root_path); +} + Result> InternalReadContext::CreateFromCoreOptions( const std::shared_ptr& context, const std::shared_ptr& table_schema, CoreOptions core_options, const std::string* path) { diff --git a/src/paimon/core/operation/internal_read_context.h b/src/paimon/core/operation/internal_read_context.h index b7218a686..43783e950 100644 --- a/src/paimon/core/operation/internal_read_context.h +++ b/src/paimon/core/operation/internal_read_context.h @@ -59,20 +59,17 @@ class InternalReadContext { }; /// Builds the context of a read a table serves: the options are the table's own, already - /// parsed - the store's `options()` - and the cache the context carries is armed on them - /// here. `path` names the root path the table was opened with, which the read reads from in - /// place of the one named by the context. + /// parsed, and the cache the context carries is armed on them here. `path` names the root path + /// the table was opened with, which the read reads from in place of the context's own. static Result> Create( const std::shared_ptr& read_context, const std::shared_ptr& table_schema, CoreOptions core_options, const std::string* path = nullptr); - /// Builds the context of a read whose options are composed on the spot rather than the - /// table's own - the fallback branch of a read, whose schema belongs to another branch, and - /// the reads a write or a lookup serve inside themselves. The options are parsed from - /// `options` here: the read's own file system is `file_system`, or the context's when null - - /// the scheme map the context carries does not take part. A cache the context carries is - /// armed on the options this parses. + /// Builds the context of a read whose options are composed on the spot rather than the table's + /// own - the fallback branch of a read, whose schema belongs to another branch, and the reads a + /// write or a lookup serve inside themselves. The read's own file system is `file_system`, or + /// the context's when null; the scheme map the context carries does not take part. static Result> Create( const std::shared_ptr& read_context, const std::shared_ptr& table_schema, @@ -80,6 +77,22 @@ class InternalReadContext { const std::shared_ptr& file_system = nullptr, const std::string* path = nullptr); + /// The read context of a fallback branch, whose schema is not the table's own: it is read + /// through a manager of that branch, and the read serves the options that schema was loaded + /// with, the branch it was read from among them. A branch the context carries is not honoured + /// here. The C++ reading of the Java `FileStoreTableFactory#createOtherBranchTable`, which + /// loads the other branch's table the same way for the fallback reader to serve. + /// @param file_system The file system the table was opened with, in place of one resolved from + /// the context. + /// @param root_path The root path the table was opened with, in place of the one the context + /// names. + /// @param table_dynamic_options The dynamic options the load merged into the table - and the + /// ones its copies accumulated. + static Result> CreateForFallbackBranch( + const std::shared_ptr& context, const std::string& branch, + const std::shared_ptr& file_system, const std::string& root_path, + const std::map& table_dynamic_options); + /// Maps score output positions in the read schema to their logical search sources. const std::map& GetIndexScoreBindings() const { return index_score_bindings_; diff --git a/src/paimon/core/operation/orphan_files_cleaner.cpp b/src/paimon/core/operation/orphan_files_cleaner.cpp index 397ba68ea..8fae06e2c 100644 --- a/src/paimon/core/operation/orphan_files_cleaner.cpp +++ b/src/paimon/core/operation/orphan_files_cleaner.cpp @@ -33,6 +33,7 @@ #include "paimon/core/operation/orphan_files_cleaner_impl.h" #include "paimon/core/schema/schema_manager.h" #include "paimon/core/schema/table_schema.h" +#include "paimon/core/utils/context_utils.h" #include "paimon/core/utils/field_mapping.h" #include "paimon/core/utils/file_store_path_factory.h" #include "paimon/core/utils/snapshot_manager.h" @@ -150,15 +151,7 @@ Result> CleanContextBuilder::Finish() { Result> OrphanFilesCleaner::Create( std::unique_ptr&& ctx) { - if (ctx == nullptr) { - return Status::Invalid("clean context is null pointer"); - } - if (ctx->GetMemoryPool() == nullptr) { - return Status::Invalid("memory pool is null pointer"); - } - if (ctx->GetExecutor() == nullptr) { - return Status::Invalid("executor is null pointer"); - } + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(ctx, "clean")); if (ctx->GetOlderThanMs() < 0) { return Status::Invalid("older than time needs to be greater than or equal to 0."); } diff --git a/src/paimon/core/schema/schema_validation.cpp b/src/paimon/core/schema/schema_validation.cpp index 3a73838d9..5d112417f 100644 --- a/src/paimon/core/schema/schema_validation.cpp +++ b/src/paimon/core/schema/schema_validation.cpp @@ -76,6 +76,29 @@ namespace { constexpr char kDeletionVectorsMergeOnRead[] = "deletion-vectors.merge-on-read"; +/// The options a copy is refused to change, mirroring the Java `SchemaManagerUtils# +/// checkAlterTableOption` over the immutable options the C++ side has constants for. They name +/// the physical layout the data on the table's path was written by: a `bucket-key` the options +/// rename, say, re-hashes the bucket pruning while the files stay hashed by the old key, so the +/// scans a copy serves would filter out the buckets its rows are actually in. The check runs over +/// every option the call gives, before the merge, so it holds on every path that merges options +/// into a table. Unlike the Java set, `merge-engine`, `bucket`, `row-tracking.enabled` and +/// `data-evolution.enabled` stay servable as dynamic options here, and `type`/`branch` are dropped +/// by `Copy` before this point. +const char* const kImmutableOptions[] = { + Options::BUCKET_KEY, + Options::BUCKET_FUNCTION_TYPE, + Options::AGGREGATION_REMOVE_RECORD_ON_DELETE, + Options::PARTIAL_UPDATE_REMOVE_RECORD_ON_DELETE, + Options::PARTIAL_UPDATE_REMOVE_RECORD_ON_SEQUENCE_GROUP, + Options::FORCE_LOOKUP, + Options::INDEX_FILE_IN_DATA_FILE_DIR, + Options::BLOB_FIELD, + Options::FALLBACK_BLOB_DESCRIPTOR_FIELD, + Options::BLOB_VIEW_FIELD, + Options::PK_CLUSTERING_OVERRIDE, +}; + bool ContainsBlobField(const std::shared_ptr& field) { if (BlobUtils::IsBlobField(field)) { return true; @@ -402,6 +425,32 @@ Status SchemaValidation::ValidateTableSchema(const TableSchema& schema) { return Status::OK(); } +Status SchemaValidation::CheckImmutableOptions( + const TableSchema& schema, const std::map& dynamic_options) { + const std::map& schema_options = schema.Options(); + for (const auto& [key, value] : dynamic_options) { + auto schema_option = schema_options.find(key); + if (schema_option != schema_options.end() && schema_option->second == value) { + continue; + } + if (std::find(std::begin(kImmutableOptions), std::end(kImmutableOptions), key) != + std::end(kImmutableOptions)) { + return Status::Invalid(fmt::format("Change '{}' is not supported yet.", key)); + } + if (key == Options::IGNORE_DELETE) { + // An unset option is one that never ignored deletes, the same default they parse to. + PAIMON_ASSIGN_OR_RAISE(bool old_ignore_delete, + OptionsUtils::GetValueFromMap(schema_options, key, false)); + PAIMON_ASSIGN_OR_RAISE(bool new_ignore_delete, + OptionsUtils::GetValueFromMap(dynamic_options, key)); + if (old_ignore_delete && !new_ignore_delete) { + return Status::Invalid("Cannot change ignore-delete from true to false."); + } + } + } + return Status::OK(); +} + Status SchemaValidation::ValidateNoDuplicateField(const std::vector& field_names, const std::string& error_message_intro) { auto duplicate_field_names = ObjectUtils::DuplicateItems(field_names); diff --git a/src/paimon/core/schema/schema_validation.h b/src/paimon/core/schema/schema_validation.h index 813aaebb4..ba169c057 100644 --- a/src/paimon/core/schema/schema_validation.h +++ b/src/paimon/core/schema/schema_validation.h @@ -51,6 +51,15 @@ class SchemaValidation { static Status ValidateTableSchema(const TableSchema& schema); + /// Whether a copy's dynamic options stay within what the data on the table's path was laid out + /// with - the C++ reading of the Java `AbstractFileStoreTable#checkImmutability`. An option + /// that says what the schema already says moves nothing, an immutable one that says anything + /// else is refused, and `ignore-delete` carries the Java check's directional rule of its own: + /// it may only ever be turned on, never back off. The immutable options themselves are the + /// Java `CoreOptions.IMMUTABLE_OPTIONS`, as far as the C++ side has constants for them. + static Status CheckImmutableOptions(const TableSchema& schema, + const std::map& dynamic_options); + /// Validates real-time options after operation-specific options have been merged with the /// table schema options. static Status ValidateRealtime(const TableSchema& schema, const CoreOptions& options); diff --git a/src/paimon/core/table/append_only_file_store_table.cpp b/src/paimon/core/table/append_only_file_store_table.cpp index cead79566..fb504ee4c 100644 --- a/src/paimon/core/table/append_only_file_store_table.cpp +++ b/src/paimon/core/table/append_only_file_store_table.cpp @@ -38,7 +38,6 @@ #include "paimon/core/table/bucket_mode.h" #include "paimon/core/table/file_store.h" #include "paimon/core/table/source/append_only_split_generator.h" -#include "paimon/core/table/source/append_only_table_read.h" #include "paimon/core/table/source/data_evolution_split_generator.h" #include "paimon/core/table/source/split_generator.h" #include "paimon/core/utils/snapshot_manager.h" @@ -60,7 +59,7 @@ Result> AppendOnlyFileStoreTable::CreateWrite( const std::shared_ptr& snapshot_manager, const std::shared_ptr& io_manager, bool ignore_previous_files) const { // append table - const std::shared_ptr& schema = store_->schema(); + const std::shared_ptr& schema = store_->GetSchema(); const bool realtime_deduplicate = !options.GetRealtimeDeduplicateKeyFields().empty(); bool need_dv_maintainer_factory = options.DeletionVectorsEnabled(); if (options.GetBucket() == -1) { @@ -139,7 +138,7 @@ Result> AppendOnlyFileStoreTable::NewFileStoreSca const std::shared_ptr& executor, const std::shared_ptr& memory_pool, const ScanContext* context) const { // TODO(liancheng.lsz): support fallback branch in scan - const std::shared_ptr& table_schema = store_->schema(); + const std::shared_ptr& table_schema = store_->GetSchema(); std::unique_ptr scan; if (core_options.DataEvolutionEnabled()) { PAIMON_ASSIGN_OR_RAISE( @@ -172,15 +171,6 @@ Result> AppendOnlyFileStoreTable::NewSplitGenera source_split_open_file_cost, bucket_mode); } -Result> AppendOnlyFileStoreTable::NewTableRead( - const std::shared_ptr& path_factory, - const std::shared_ptr& internal_context, - const std::shared_ptr& memory_pool, - const std::shared_ptr& executor) const { - return std::make_unique(path_factory, internal_context, memory_pool, - executor); -} - CommitScanner::ScanSupplier AppendOnlyFileStoreTable::NewCommitScanSupplier( const std::shared_ptr& snapshot_manager, const std::shared_ptr& manifest_list, @@ -190,7 +180,7 @@ CommitScanner::ScanSupplier AppendOnlyFileStoreTable::NewCommitScanSupplier( // The schema manager the table carries, which the scans this supplier hands out read the // schema through. const std::shared_ptr& schema_manager = GetSchemaManager(); - const std::shared_ptr& table_schema = store_->schema(); + const std::shared_ptr& table_schema = store_->GetSchema(); return [snapshot_manager, schema_manager, manifest_list, manifest_file, table_schema, arrow_schema, options, executor, pool](const std::shared_ptr& scan_filter) -> Result> { diff --git a/src/paimon/core/table/append_only_file_store_table.h b/src/paimon/core/table/append_only_file_store_table.h index 499431eb0..9b4af46ae 100644 --- a/src/paimon/core/table/append_only_file_store_table.h +++ b/src/paimon/core/table/append_only_file_store_table.h @@ -63,12 +63,6 @@ class AppendOnlyFileStoreTable : public FileStoreTable { Result> NewSplitGenerator( const CoreOptions& core_options) const override; - Result> NewTableRead( - const std::shared_ptr& path_factory, - const std::shared_ptr& internal_context, - const std::shared_ptr& memory_pool, - const std::shared_ptr& executor) const override; - CommitScanner::ScanSupplier NewCommitScanSupplier( const std::shared_ptr& snapshot_manager, const std::shared_ptr& manifest_list, diff --git a/src/paimon/core/table/file_store.cpp b/src/paimon/core/table/file_store.cpp index 472626a01..fe8b092c7 100644 --- a/src/paimon/core/table/file_store.cpp +++ b/src/paimon/core/table/file_store.cpp @@ -47,6 +47,19 @@ namespace paimon { +namespace { + +/// Whether an option key takes part in file-system resolution: the backend `file-system` picks, +/// an `fs.` access key carries, or a native S3 key names - `s3.` and its `s3a.` spelling, the +/// prefixes the S3 file system consumes alongside `fs.s3.`/`fs.s3a.`. The scheme-map keys a +/// context carries are the same set. +bool IsFileSystemResolutionKey(const std::string& key) { + return key == Options::FILE_SYSTEM || key.rfind("fs.", 0) == 0 || key.rfind("s3.", 0) == 0 || + key.rfind("s3a.", 0) == 0; +} + +} // namespace + Result> FileStore::Create( const std::shared_ptr& file_system, const std::string& root_path, const std::shared_ptr& schema, const std::string& branch, @@ -97,6 +110,42 @@ Result> FileStore::Create( return store; } +bool FileStore::FileSystemResolutionMayChange( + const TableSchema& schema, const std::map& dynamic_options) { + const std::map& schema_options = schema.Options(); + for (const auto& [key, value] : dynamic_options) { + if (!IsFileSystemResolutionKey(key)) { + continue; + } + auto schema_option = schema_options.find(key); + if (schema_option == schema_options.end() || schema_option->second != value) { + return true; + } + } + return false; +} + +bool FileStore::FileSystemResolutionAgrees(const std::map& left, + const std::map& right) { + for (const auto& [key, value] : left) { + if (IsFileSystemResolutionKey(key)) { + auto right_option = right.find(key); + if (right_option == right.end() || right_option->second != value) { + return false; + } + } + } + for (const auto& [key, value] : right) { + if (IsFileSystemResolutionKey(key)) { + auto left_option = left.find(key); + if (left_option == left.end() || left_option->second != value) { + return false; + } + } + } + return true; +} + FileStore::FileStore(const std::shared_ptr& file_system, const std::string& root_path, const std::shared_ptr& schema, const std::string& branch, const std::shared_ptr& schema_manager, diff --git a/src/paimon/core/table/file_store.h b/src/paimon/core/table/file_store.h index 66ccc09c2..e343ec2c4 100644 --- a/src/paimon/core/table/file_store.h +++ b/src/paimon/core/table/file_store.h @@ -47,46 +47,34 @@ class ManifestList; class TableSchema; /// The storage a managed table is served through: the file system, root path, schema, branch, -/// options and the schema and snapshot managers it was loaded with - the state Java Paimon's -/// `AbstractFileStore` carries, parsed once here rather than per operator. `FileStoreTable` hands -/// the state it was loaded with to one of these and asks it back for the pieces every operator -/// opens with. +/// options and managers it was loaded with - the state Java Paimon's `AbstractFileStore` carries, +/// parsed once here rather than per operator. `FileStoreTable` hands the state it was loaded with +/// to one of these and asks it back for the pieces every operator opens with. /// -/// The options are the ones the table was loaded with on this branch - the schema's own, or -/// the ones a load merged over them - the base every operator serves: `NewXxx` takes no -/// options of its own beyond them. A cache is not part of them - `WithCache()` arms a copy an -/// operator makes - so the factories below take the options of the call, not the store's own. -/// -/// The factory methods build a fresh instance on every call, never a cached one: a path factory -/// names each file it builds, and a snapshot manager a write or a commit arms with a catalog -/// loader is not one a scan should be served through afterwards. +/// Every operator serves the options the table was loaded with, so the factories below take the +/// options of the call: a cache is not part of the table's, `WithCache()` arms a copy an operator +/// makes. They build a fresh instance on every call, never a cached one - a snapshot manager a +/// write arms with a catalog loader is not one a scan should be served through afterwards. class FileStore { public: /// @param file_system The file system the caller opened the table through, which wins over - /// the access configuration the schema's options may carry when it is not null. A - /// null one hands the resolution to that configuration: the table is then served by - /// the file system its options name - the way a load that only had a path and a - /// schema to read names it - and by the default local one when they name none. + /// the access configuration the schema's options may carry when it is not null. A null + /// one hands the resolution to that configuration, and to the default local file system + /// when the options name none. /// @param root_path Root path of the table in the file system. - /// @param schema The schema the table was loaded with, whose partition keys name the - /// partitions the paths below are built for and whose options `options()` parses. It - /// is null when the caller had no schema; the operators then report that none was - /// found. + /// @param schema The schema the table was loaded with, whose partition keys name the partitions + /// the paths below are built for and whose options `GetOptions()` parses. Null when the + /// caller had none, which the operators then report. /// @param branch The branch the store serves, normalized here. - /// @param schema_manager A schema manager to carry instead of building one - the one a - /// context dispatch read the schema through, whose cache the operators then serve. - /// @param snapshot_manager A snapshot manager to carry instead of building one. - /// A null manager is built over `root_path` on `branch`, so the operators always read - /// through the managers the store carries. + /// @param schema_manager A schema manager to carry instead of building one - the one a context + /// dispatch read the schema through, whose cache the operators then serve. + /// @param snapshot_manager A snapshot manager to carry instead of building one; a null one is + /// built over `root_path` on `branch`. /// @param fs_scheme_map The scheme-to-identifier map the caller's contexts resolve custom - /// schemes through. It takes part in building the store's options the way it took - /// part in reading the schema: without it a schema naming a custom scheme would - /// resolve to the default backend once its operators run. It is kept for a copy to - /// hand over again, so the copy resolves as this store does. + /// schemes through, kept so a copy resolves as this store does. /// - /// Building the store parses the schema's options, so it fails when they do not parse: the - /// error is the caller's to see, which is why a store is built through this factory rather - /// than the constructor. + /// Building the store parses the schema's options, so it fails when they do not parse - which + /// is why a store is built through this factory rather than the constructor. static Result> Create( const std::shared_ptr& file_system, const std::string& root_path, const std::shared_ptr& schema, const std::string& branch, @@ -94,69 +82,74 @@ class FileStore { const std::shared_ptr& snapshot_manager = nullptr, const std::map& fs_scheme_map = {}); - /// The branch this store serves, as resolved when it was built. - const std::string& branch() const { + /// Whether a copy's dynamic options change the file system its store resolves to. If they do, + /// the managers of the table it was copied from are not its to inherit: built over that + /// table's resolution, they would keep reading metadata through the backend the copy just + /// left. + static bool FileSystemResolutionMayChange( + const TableSchema& schema, const std::map& dynamic_options); + + /// Whether two options maps resolve to the same file system: every resolution key has to + /// agree, a missing one on both sides included. + static bool FileSystemResolutionAgrees(const std::map& left, + const std::map& right); + + const std::string& CurrentBranch() const { return branch_; } - /// The schema manager the operators read through. It carries the schema cache of `root_path` - /// on `branch()`. - const std::shared_ptr& schema_manager() const { + /// The schema manager the operators read through, carrying the schema cache of `root_path` on + /// `CurrentBranch()`. + const std::shared_ptr& GetSchemaManager() const { return schema_manager_; } /// The snapshot manager a scan reads through. A write or a commit builds its own: see /// `NewSnapshotManager`. - const std::shared_ptr& snapshot_manager() const { + const std::shared_ptr& GetSnapshotManager() const { return snapshot_manager_; } - /// The schema the table was loaded with, which every operator serves rather than reading the - /// schema again. It is null when the caller had no schema. - const std::shared_ptr& schema() const { + /// The schema the table was loaded with, null when the caller had none. + const std::shared_ptr& GetSchema() const { return schema_; } - /// The file system the table's data is accessed through, as the table was opened with it: - /// the one the options resolve - a schema carrying access configuration names its own, - /// which the file system the caller opened the table through wins over - rather than the - /// one that call handed over, which `specified_file_system()` hands back untouched. - const std::shared_ptr& file_system() const { + /// The file system the table's data is accessed through: the one the options resolve, which + /// the file system the caller opened the table through wins over. + const std::shared_ptr& GetFileSystem() const { return file_system_; } - /// The file system the caller opened the table through, as it was handed to `Create`: null - /// when the call had none, so a copy of the table resolves from its own options again and - /// the runtime ones it merges in - a file-system option among them - can speak. - const std::shared_ptr& specified_file_system() const { + /// The file system the caller opened the table through, as handed to `Create`: null when the + /// call had none, so a copy resolves from its own options again. + const std::shared_ptr& GetSpecifiedFileSystem() const { return specified_file_system_; } /// The scheme-to-identifier map the options were resolved with, kept for a copy to resolve - /// through again: see `Create`'s `fs_scheme_map`. - const std::map& fs_scheme_map() const { + /// through again. + const std::map& GetFsSchemeMap() const { return fs_scheme_map_; } - /// The root path of the table in its file system, as the table was opened with it. - const std::string& root_path() const { + const std::string& GetRootPath() const { return root_path_; } - /// The options the schema was loaded with on `branch()`, which every operator serves. They - /// exist only over a schema: an operator reads them after reporting that no schema was - /// found when there is none. - const CoreOptions& options() const { + /// The options the schema was loaded with on `CurrentBranch()`. They exist only over a schema: + /// an operator reads them after reporting that none was found when there is not. + const CoreOptions& GetOptions() const { assert(core_options_.has_value()); return *core_options_; } /// The path factory an operator builds its metadata paths through: over this store's root - /// path and the partition keys of its schema - which no schema change can alter, so a read - /// serving a fallback branch's schema still names the same partitions. The file format - /// identifier is the caller's to name: a write names the format its level-0 files are - /// written in, which the options may set per level, while a scan, a read and a commit name - /// the table's format. + /// path and the partition keys of its schema, which no schema change can alter, so an + /// operator serving a schema read after the table was loaded still names the same partitions. + /// The file format identifier is the caller's to name: a write names the format its level-0 + /// files are written in, which the options may set per level, while a scan, a read and a + /// commit name the table's format. Result> NewPathFactory( const std::shared_ptr& arrow_schema, const CoreOptions& options, const std::string& file_format_identifier, @@ -164,46 +157,39 @@ class FileStore { /// The snapshot manager a write or a commit reads snapshots through, over this store's root /// path and branch. The file system is the caller's to name, from the options its operator - /// serves: a table opened without one has its options build the resolving file system. A - /// catalog that version-manages the table hands its snapshots out itself, so its loader is - /// attached, the catalog captured to keep the versioned interface - and the loader with it - /// - alive. Such a catalog names the table by an identifier, which the context must have - /// supplied. + /// serves. A catalog that version-manages the table hands its snapshots out itself, so its + /// loader is attached and the catalog captured to keep it alive; such a catalog names the + /// table by an identifier, which the context must have supplied. Result> NewSnapshotManager( const CoreOptions& options, const std::shared_ptr& catalog, const std::optional& catalog_identifier) const; - /// The manifest list a scan or a commit reads the table's manifest entries through, over - /// the caller's path factory: an operator's manifests sit in the paths its path factory - /// names. + /// The manifest list a scan or a commit reads the table's manifest entries through. Result> NewManifestList( const CoreOptions& options, const std::shared_ptr& path_factory, const std::shared_ptr& memory_pool) const; - /// The manifest file a scan or a commit reads a single manifest's entries through, over - /// the caller's path factory and the partition schema its entries are pruned by. + /// The manifest file a scan or a commit reads a single manifest's entries through, pruned by + /// `partition_schema`. Result> NewManifestFile( const CoreOptions& options, const std::shared_ptr& path_factory, const std::shared_ptr& partition_schema, const std::shared_ptr& memory_pool) const; /// The index manifest file a write, a scan or a commit reads the table's index manifest - /// through, over the caller's path factory. + /// through. Result> NewIndexManifestFile( const CoreOptions& options, const std::shared_ptr& path_factory, const std::shared_ptr& memory_pool) const; - /// The index file handler a write's deletion vectors and indexes are maintained through, or - /// a scan reads them through: the index manifest file above and the path factories over the - /// caller's path factory, in one handler. + /// The index file handler a write maintains its deletion vectors and indexes through, or a + /// scan reads them through. Result> NewIndexFileHandler( const CoreOptions& options, const std::shared_ptr& path_factory, const std::shared_ptr& memory_pool) const; /// The partition computer a commit names the partitions it writes with, over the partition - /// keys of this store's schema - which no schema change can alter, so a commit serving a - /// schema read after the table was loaded still names the same partitions - and the - /// caller's arrow schema and options. + /// keys of this store's schema and the caller's arrow schema. Result> NewPartitionComputer( const std::shared_ptr& arrow_schema, const CoreOptions& options, const std::shared_ptr& memory_pool) const; @@ -216,20 +202,14 @@ class FileStore { const std::map& fs_scheme_map); std::shared_ptr file_system_; - /// The file system the caller opened the table through, as handed to `Create` - null when - /// the call had none - kept for a copy to hand over again: what `file_system_` holds is - /// resolved, and would pin the copy to this table's resolution. std::shared_ptr specified_file_system_; - /// The scheme-to-identifier map the options were resolved with, kept for a copy to resolve - /// through again. std::map fs_scheme_map_; std::string root_path_; std::shared_ptr schema_; std::string branch_; std::shared_ptr schema_manager_; std::shared_ptr snapshot_manager_; - /// The options of `schema_` on `branch_`, or unset when there is no schema. `Create` parses - /// them: the constructor cannot report a parse failure. + /// The options of `schema_` on `branch_`, unset when there is no schema. std::optional core_options_; }; diff --git a/src/paimon/core/table/file_store_table.cpp b/src/paimon/core/table/file_store_table.cpp index 6a52903b1..7e68b3567 100644 --- a/src/paimon/core/table/file_store_table.cpp +++ b/src/paimon/core/table/file_store_table.cpp @@ -19,13 +19,11 @@ #include "paimon/core/table/file_store_table.h" -#include #include #include #include #include #include -#include #include "fmt/format.h" #include "paimon/catalog/catalog.h" @@ -34,14 +32,10 @@ #include "paimon/common/predicate/predicate_validator.h" #include "paimon/common/types/data_field.h" #include "paimon/common/utils/binary_row_partition_computer.h" -#include "paimon/common/utils/checked_cast.h" #include "paimon/common/utils/options_utils.h" -#include "paimon/common/utils/path_util.h" #include "paimon/common/utils/sensitive_config_utils.h" #include "paimon/common/utils/string_utils.h" -#include "paimon/core/catalog/catalog_snapshot_commit.h" #include "paimon/core/catalog/catalog_utils.h" -#include "paimon/core/catalog/renaming_snapshot_commit.h" #include "paimon/core/catalog/snapshot_commit.h" #include "paimon/core/catalog/version_managed_catalog.h" #include "paimon/core/core_options.h" @@ -54,7 +48,6 @@ #include "paimon/core/operation/file_store_commit_impl.h" #include "paimon/core/operation/file_store_scan.h" #include "paimon/core/operation/internal_read_context.h" -#include "paimon/core/options/table_type.h" #include "paimon/core/realtime/realtime_context_impl.h" #include "paimon/core/schema/schema_manager.h" #include "paimon/core/schema/schema_validation.h" @@ -62,20 +55,22 @@ #include "paimon/core/snapshot.h" #include "paimon/core/table/append_only_file_store_table.h" #include "paimon/core/table/file_store.h" +#include "paimon/core/table/file_store_table_factory.h" #include "paimon/core/table/primary_key_file_store_table.h" +#include "paimon/core/table/source/append_only_table_read.h" #include "paimon/core/table/source/data_evolution_batch_scan.h" #include "paimon/core/table/source/data_table_batch_scan.h" #include "paimon/core/table/source/data_table_stream_scan.h" #include "paimon/core/table/source/fallback_table_read.h" +#include "paimon/core/table/source/key_value_table_read.h" #include "paimon/core/table/source/primary_key_index_batch_scan.h" #include "paimon/core/table/source/read_optimized_scan_options.h" #include "paimon/core/table/source/realtime_table_scan.h" #include "paimon/core/table/source/snapshot/snapshot_reader.h" -#include "paimon/core/table/system/system_table.h" #include "paimon/core/utils/branch_manager.h" +#include "paimon/core/utils/context_utils.h" #include "paimon/core/utils/field_mapping.h" #include "paimon/core/utils/file_store_path_factory.h" -#include "paimon/core/utils/primary_key_table_utils.h" #include "paimon/core/utils/snapshot_manager.h" #include "paimon/file_store_commit.h" #include "paimon/file_store_write.h" @@ -88,7 +83,6 @@ #include "paimon/scan_context.h" #include "paimon/schema/schema.h" #include "paimon/status.h" -#include "paimon/table/format/format_table.h" #include "paimon/table/source/table_read.h" #include "paimon/table/source/table_scan.h" #include "paimon/write_context.h" @@ -102,477 +96,9 @@ namespace paimon { class Executor; class MemoryPool; -namespace { - -/// The identifier a table path names when no catalog name goes with it, the C++ reading of -/// the Java `SchemaManagerUtils#identifierFromPath` over `ignoreIfUnknownDatabase = true`: the -/// directory above the table's names the database when it carries the `.db` suffix, the way a -/// catalog lays its databases out, and a path that names no database - a table sitting directly -/// under the root, say - leaves the database unknown, which is all a caller that only knows the -/// directory can say. -Identifier IdentifierFromPath(const std::string& table_path, const std::string& branch) { - const std::vector paths = StringUtils::Split(table_path, "/"); - if (paths.size() < 2) { - return Identifier(Identifier::kUnknownDatabase, - paths.empty() ? std::string() : paths.front()); - } - const std::string& database = paths[paths.size() - 2]; - const std::string::size_type index = database.rfind(Catalog::DB_SUFFIX); - if (index == std::string::npos) { - return Identifier(Identifier::kUnknownDatabase, paths.back(), branch); - } - return Identifier(database.substr(0, index), paths.back(), branch); -} - -/// Recognises a format table behind `table_path`, or returns null when it is not one. -/// -/// The scan, read, write and commit entry points below all take a path and have to answer the -/// same question before they can dispatch, so one place answers it and they cannot disagree. -/// Null is also the answer when no schema was found under the path at all: what to say about a -/// table that is not there is the managed path's to decide, and it says it in more detail. -/// -/// The table type is read from the schema, never from `options`: `type` is structural, and one -/// call's options must not decide what kind of table this is. `branch` names the branch the -/// schema was read from: anything but the main branch is refused once the schema turns out to be -/// a format table's, which has no branches to read. -Result> TryLoadFormatTable( - const std::shared_ptr& file_system, const std::string& table_path, - const std::string& branch, const std::map& options, - const std::shared_ptr& table_schema) { - if (table_schema == nullptr) { - return std::shared_ptr(); - } - // From the schema alone: see `FormatTable::Create()`. - PAIMON_ASSIGN_OR_RAISE(bool is_format_table, - TableTypeDefine::IsFormatTable(table_schema->Options())); - if (!is_format_table) { - return std::shared_ptr(); - } - // A format table has no branches, so it is refused rather than served from the table path - // under another branch's name. See `SchemaValidation::ValidateFormatTableSchema()`. - if (BranchManager::NormalizeBranch(branch) != BranchManager::DEFAULT_MAIN_BRANCH) { - return Status::Invalid( - fmt::format("a format table has no branches, so {} cannot be read on branch '{}'", - table_path, branch)); - } - // A context names a path, never a catalog identifier, so the table is known by the - // directory it sits in - which names the database too, when it carries the `.db` suffix a - // catalog lays its databases out with. - Identifier identifier = IdentifierFromPath(table_path, branch); - return FormatTable::Create(file_system, table_path, identifier, - checked_pointer_cast(table_schema), - /*location_carries_paimon_metadata=*/true, options); -} - -/// Whether an option of a copy names the file system its store resolves: the backend -/// `file-system` picks, or the access configuration a `fs.` key carries. A copy whose dynamic -/// options change one of these resolves to another file system, so the managers of the table it -/// was copied from - built over that table's resolution - are not its to inherit: they would -/// keep reading metadata through the backend the copy just left. -bool FileSystemResolutionMayChange(const TableSchema& schema, - const std::map& dynamic_options) { - const std::map& schema_options = schema.Options(); - for (const auto& [key, value] : dynamic_options) { - if (key != Options::FILE_SYSTEM && key.rfind("fs.", 0) != 0) { - continue; - } - auto schema_option = schema_options.find(key); - if (schema_option == schema_options.end() || schema_option->second != value) { - return true; - } - } - return false; -} - -/// The options a copy is refused to change, mirroring the Java `SchemaManagerUtils# -/// checkAlterTableOption` over the immutable options the C++ side has constants for. These name -/// the physical layout and the file set the data on the table's path was written by: a -/// `bucket-key` the options rename, say, re-hashes the bucket pruning while the files stay -/// hashed by the old key, so the scans a copy serves would filter out the buckets its rows are -/// actually in. The check runs over every option the call gives, before the merge, so it holds -/// on every path that merges options into a table - `Copy` itself and the load a context -/// drives alike. The differences from the Java set: `merge-engine`, `bucket`, -/// `row-tracking.enabled` and `data-evolution.enabled` stay servable as dynamic options here - -/// the C++ tests load tables through context options that move them - and `type`/`branch` are -/// dropped by `Copy` before this point. -const char* const kImmutableOptions[] = { - Options::BUCKET_KEY, - Options::BUCKET_FUNCTION_TYPE, - Options::AGGREGATION_REMOVE_RECORD_ON_DELETE, - Options::PARTIAL_UPDATE_REMOVE_RECORD_ON_DELETE, - Options::PARTIAL_UPDATE_REMOVE_RECORD_ON_SEQUENCE_GROUP, - Options::FORCE_LOOKUP, - Options::INDEX_FILE_IN_DATA_FILE_DIR, - Options::BLOB_FIELD, - Options::FALLBACK_BLOB_DESCRIPTOR_FIELD, - Options::BLOB_VIEW_FIELD, - Options::PK_CLUSTERING_OVERRIDE, -}; - -/// Whether the dynamic options of a copy stay within what the data on the table's path was -/// laid out with - the C++ reading of the Java `AbstractFileStoreTable#checkImmutability`: -/// every option that says what the schema already says moves nothing, an immutable one that -/// says anything else is refused, and an `ignore-delete` carries the directional rule of its -/// own the Java check carries too: it may only ever be turned on, never back off. -Status CheckImmutableOptions(const TableSchema& schema, - const std::map& dynamic_options) { - const std::map& schema_options = schema.Options(); - for (const auto& [key, value] : dynamic_options) { - auto schema_option = schema_options.find(key); - if (schema_option != schema_options.end() && schema_option->second == value) { - continue; - } - if (std::find(std::begin(kImmutableOptions), std::end(kImmutableOptions), key) != - std::end(kImmutableOptions)) { - return Status::Invalid(fmt::format("Change '{}' is not supported yet.", key)); - } - if (key == Options::IGNORE_DELETE) { - // An unset option is one that never ignored deletes, the same default the options - // parse to. - PAIMON_ASSIGN_OR_RAISE(bool old_ignore_delete, - OptionsUtils::GetValueFromMap(schema_options, key, false)); - PAIMON_ASSIGN_OR_RAISE(bool new_ignore_delete, - OptionsUtils::GetValueFromMap(dynamic_options, key)); - if (old_ignore_delete && !new_ignore_delete) { - return Status::Invalid("Cannot change ignore-delete from true to false."); - } - } - } - return Status::OK(); -} - -/// The options a context carries, merged into the table the load hands back: a table is the -/// product of the load, so the options speak there - the way `Copy` merges them, over the ones -/// the schema carries - and `NewXxx` on the product serves them without merging again. A -/// caller holding a table already - a load that is over - changes it through `Copy` itself, -/// and an option that contradicts the table that way is refused. -Result> WithContextOptions( - const std::shared_ptr
& table, const std::shared_ptr& schema, - const std::map& options) { - if (table == nullptr || schema == nullptr || options.empty()) { - return table; - } - const std::map& schema_options = schema->Options(); - for (const auto& [key, value] : options) { - auto schema_option = schema_options.find(key); - // A key the schema does not carry, or carries differently: the merge has something to - // do. Every key agreeing with the schema leaves the table the load built as it is. - if (schema_option == schema_options.end() || schema_option->second != value) { - return table->Copy(options); - } - } - return table; -} - -/// The tail every `TableOf` ends with: the schema one of them read says what the path names. A -/// format table is handed back as the `FormatTable` it is, everything else as the -/// `FileStoreTable` a managed table is, carrying the schema for its `NewXxx` to build from. A -/// null schema is handed over as it is: what to say about a table that is not there is the -/// managed path's to decide, and it says it in more detail. -/// -/// @param file_system The file system the schema above was read through, which a format table -/// is served by - it has no schema options of its own to name another one. -/// @param explicit_file_system The file system the context supplied, which a managed table is -/// opened through when there is one. A null one leaves the table to resolve from the -/// options its schema carries: access configuration the schema was stored with then -/// names the files it serves, rather than being pinned to the one this load happened to -/// read the schema through. -Result> TableOfLoadedSchema( - const std::shared_ptr& file_system, - const std::shared_ptr& explicit_file_system, const std::string& path, - const std::string& branch, const std::map& options, - const std::shared_ptr& schema_manager, - const std::shared_ptr& latest_schema, const std::optional& identifier, - const std::string& uuid = "", const std::map& fs_scheme_map = {}) { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr format_table, - TryLoadFormatTable(file_system, path, branch, options, latest_schema)); - if (format_table != nullptr) { - return std::shared_ptr
(std::move(format_table)); - } - // A context names a path, never a catalog identifier, so the table is known by the - // directory it sits in - which names the database too, when it carries the `.db` suffix a - // catalog lays its databases out with. The branch the schema was read on travels the same - // way, folded into the object name. - Identifier table_identifier = identifier.value_or(IdentifierFromPath(path, branch)); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table, - FileStoreTable::Create(explicit_file_system, path, latest_schema, - table_identifier, uuid, branch, schema_manager, - /*snapshot_manager=*/nullptr, fs_scheme_map)); - return WithContextOptions(table, latest_schema, options); -} - -/// The managed table a context names behind a path: the schema is read through a manager of the -/// context's branch - handed over instead when the caller already read it - and what the schema -/// says is there decides the table `TableOfLoadedSchema` hands back. -/// -/// @param schema_json The schema of the table, when the caller already read it, or null when it -/// has to be read here. It is honoured on the main branch only: a branch publishes its own -/// under `branch/branch-`, which is where the read below goes. -/// @param explicit_file_system The file system the context supplied, which the managed table the -/// schema describes is opened through - see `TableOfLoadedSchema`. -Result> ManagedTableOf( - const std::shared_ptr& file_system, - const std::shared_ptr& explicit_file_system, const std::string& path, - const std::string& branch, const std::map& options, - const std::optional& schema_json, const std::optional& identifier, - const std::string& uuid = "", const std::map& fs_scheme_map = {}) { - auto schema_manager = std::make_shared(file_system, path, branch); - std::shared_ptr latest_schema; - if (branch == BranchManager::DEFAULT_MAIN_BRANCH && schema_json) { - // Handing the schema over only saves reading it. A context always names a table path and - // this library writes the metadata under it, so `schema` and `branch` stay metadata. - PAIMON_ASSIGN_OR_RAISE(latest_schema, TableSchema::CreateFromJson(schema_json.value())); - } else { - PAIMON_ASSIGN_OR_RAISE(std::optional> latest, - schema_manager->Latest()); - if (latest) { - latest_schema = *latest; - } - } - return TableOfLoadedSchema(file_system, explicit_file_system, path, branch, options, - schema_manager, latest_schema, identifier, uuid, fs_scheme_map); -} - -/// The identifier a catalog knows the table by, on `branch`: a non-main branch is folded into -/// the identifier, so the catalog is asked for the table on that branch rather than the main one. -Result> MakeBranchCatalogIdentifier( - const std::optional& identifier, const std::string& branch) { - if (!identifier) { - return std::optional(); - } - PAIMON_ASSIGN_OR_RAISE(Identifier branch_identifier, - CatalogUtils::BranchIdentifier(identifier.value(), branch)); - return std::optional(std::move(branch_identifier)); -} - -/// The schema a catalog may hold for the table, read from it on the main branch only and null -/// otherwise. A branch publishes its own schema under `branch/branch-`, which is where -/// `ManagedTableOf` reads instead, so a catalog answering for no branch schema does not fail the -/// `kind` operation to one. `kind` also names the operator the missing-identifier message speaks -/// of. -Result> CatalogMainBranchSchemaJson( - const std::shared_ptr& catalog, const std::optional& catalog_identifier, - const std::string& branch, const char* kind) { - if (catalog == nullptr) { - return std::optional(); - } - if (!catalog_identifier) { - return Status::Invalid(fmt::format("a catalog {} requires a table identifier", kind)); - } - if (BranchManager::IsMainBranch(branch)) { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, - catalog->LoadTableSchema(catalog_identifier.value())); - PAIMON_ASSIGN_OR_RAISE(std::string schema_json, schema->GetJsonSchema()); - return std::optional(std::move(schema_json)); - } - return std::optional(); -} - -/// The table a scan context names, read from what the context carries: a format table the -/// builder was handed, a system table behind a `T$system` path, or the table - format or -/// managed - the schema under the context's path says is there. -Result> TableOf(const ScanContext& ctx) { - // A table the caller already loaded says what it is, so nothing is read to find out. - if (ctx.GetFormatTable() != nullptr) { - return std::shared_ptr
(ctx.GetFormatTable()); - } - PAIMON_ASSIGN_OR_RAISE(CoreOptions tmp_options, - CoreOptions::FromMap(ctx.GetOptions(), ctx.GetSpecificFileSystem(), {})); - PAIMON_ASSIGN_OR_RAISE(std::optional system_table_path, - SystemTableLoader::TryParsePath(ctx.GetPath())); - if (system_table_path) { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr system_table, - SystemTableLoader::LoadFromPath(tmp_options.GetFileSystem(), - ctx.GetPath(), ctx.GetOptions())); - return std::shared_ptr
(std::move(system_table)); - } - // A format table is planned by listing directories, so it never reaches the manifest path - // below. One `TableScan` interface serves both, as Java Paimon serves both through one - // `ReadBuilder`. - const std::string branch = BranchManager::NormalizeBranch(tmp_options.GetBranch()); - return ManagedTableOf(tmp_options.GetFileSystem(), ctx.GetSpecificFileSystem(), ctx.GetPath(), - branch, ctx.GetOptions(), ctx.GetSpecificTableSchema(), - ctx.GetIdentifier()); -} - -/// The table a read context names, read from what the context carries: a format table the -/// builder was handed, a system table behind a `T$system` path, or the table - format or -/// managed - the schema under the context's path says is there. -Result> TableOf(const ReadContext& ctx) { - // A table the caller already loaded says what it is, so nothing is read to find out. - if (ctx.GetFormatTable() != nullptr) { - return std::shared_ptr
(ctx.GetFormatTable()); - } - PAIMON_ASSIGN_OR_RAISE(CoreOptions tmp_core_options, - CoreOptions::FromMap(ctx.GetOptions(), ctx.GetSpecificFileSystem(), - ctx.GetFileSystemSchemeToIdentifierMap())); - PAIMON_ASSIGN_OR_RAISE(std::optional system_table_path, - SystemTableLoader::TryParsePath(ctx.GetPath())); - if (system_table_path) { - if (PredicateUtils::ContainsFullTextMatch(ctx.GetPredicate())) { - return Status::NotImplemented( - "FullTextMatch predicate execution is not implemented yet"); - } - if (ctx.HasVectorOrFullTextSearch()) { - return Status::NotImplemented( - "VectorSearch or FullTextSearch is not supported for system tables"); - } - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr system_table, - SystemTableLoader::LoadFromPath(tmp_core_options.GetFileSystem(), - ctx.GetPath(), ctx.GetOptions())); - return std::shared_ptr
(std::move(system_table)); - } - - // A format table has no manifest and a file another engine wrote carries no field ids, so - // which files there are and how a row is put back together is its own. How a file is opened - // is not: both paths go through `DataFileReaderFactory`. - const std::string branch = ctx.GetBranch(); - // The scheme map the schema was read through, so the table it describes resolves custom - // schemes the same way - the data belongs to the metadata, both sit behind the map. - return ManagedTableOf(tmp_core_options.GetFileSystem(), ctx.GetSpecificFileSystem(), - ctx.GetPath(), branch, ctx.GetOptions(), ctx.GetSpecificTableSchema(), - ctx.GetIdentifier(), /*uuid=*/"", - ctx.GetFileSystemSchemeToIdentifierMap()); -} - -/// The table a write context names, read from what the context carries: a format table the -/// builder was handed, or the table - format or managed - the schema under the context's root -/// path says is there. -Result> TableOf(const WriteContext& ctx) { - // A table the caller already loaded says what it is, so nothing is read to find out. - if (ctx.GetFormatTable() != nullptr) { - return std::shared_ptr
(ctx.GetFormatTable()); - } - PAIMON_RETURN_NOT_OK(CheckVersionManagementImplemented(ctx.GetCatalog())); - std::shared_ptr specific_fs = ctx.GetSpecificFileSystem(); - if (specific_fs == nullptr && ctx.GetFileSystemSchemeToIdentifierMap().empty() && - ctx.GetCatalog() != nullptr) { - if (!ctx.GetIdentifier()) { - return Status::Invalid("a catalog write requires a table identifier"); - } - // A catalog issuing per-table temporary credentials only hands them out through - // GetTableFileSystem, so the write uses the credentials of this table. - PAIMON_ASSIGN_OR_RAISE(specific_fs, - ctx.GetCatalog()->GetTableFileSystem(ctx.GetIdentifier().value())); - } - PAIMON_ASSIGN_OR_RAISE(CoreOptions tmp_options, - CoreOptions::FromMap(ctx.GetOptions(), specific_fs, - ctx.GetFileSystemSchemeToIdentifierMap())); - std::string branch = ctx.GetBranch(); - PAIMON_ASSIGN_OR_RAISE(std::optional catalog_identifier, - MakeBranchCatalogIdentifier(ctx.GetIdentifier(), branch)); - // Only the main branch may have a current schema the catalog alone holds. A branch publishes - // its own under `branch/branch-`, which is where the load below and a read of the branch - // both read it, so a catalog answering for no branch schema does not fail a write to one. - PAIMON_ASSIGN_OR_RAISE( - std::optional catalog_table_schema, - CatalogMainBranchSchemaJson(ctx.GetCatalog(), catalog_identifier, branch, "write")); - // A format table writes plain data files into a directory, so it never reaches the manifest - // path below. - return ManagedTableOf(tmp_options.GetFileSystem(), specific_fs, ctx.GetRootPath(), branch, - ctx.GetOptions(), catalog_table_schema, ctx.GetIdentifier(), /*uuid=*/"", - ctx.GetFileSystemSchemeToIdentifierMap()); -} - -/// The table a commit context names, read from what the context carries: a format table the -/// builder was handed, or the table - format or managed - the schema under the context's root -/// path says is there. -Result> TableOf(const CommitContext& ctx) { - // A table the caller already loaded says what it is, so nothing is read to find out. - if (ctx.GetFormatTable() != nullptr) { - return std::shared_ptr
(ctx.GetFormatTable()); - } - PAIMON_RETURN_NOT_OK(CheckVersionManagementImplemented(ctx.GetCatalog())); - // Use catalog credentials unless the caller supplied a file system. - std::shared_ptr specific_fs = ctx.GetSpecificFileSystem(); - if (specific_fs == nullptr && ctx.GetCatalog() != nullptr) { - if (!ctx.GetIdentifier()) { - return Status::Invalid("a catalog commit requires a table identifier"); - } - // Use the credentials of this table, which a catalog issuing per-table temporary - // ones only hands out through GetTableFileSystem. - PAIMON_ASSIGN_OR_RAISE(specific_fs, - ctx.GetCatalog()->GetTableFileSystem(ctx.GetIdentifier().value())); - } - PAIMON_ASSIGN_OR_RAISE(CoreOptions tmp_options, - CoreOptions::FromMap(ctx.GetOptions(), specific_fs)); - const std::string& root_path = ctx.GetRootPath(); - // The branch the context was built for, already validated by `CommitContextBuilder`. - PAIMON_ASSIGN_OR_RAISE(std::string branch, BranchManager::ResolveBranch( - ctx.GetIdentifier(), ctx.GetOptions(), - /*explicit_branch=*/std::nullopt, "commit")); - PAIMON_ASSIGN_OR_RAISE(std::optional catalog_identifier, - MakeBranchCatalogIdentifier(ctx.GetIdentifier(), branch)); - // Catalog-managed schemas may be absent from the table directory, but only on the main branch: - // a branch publishes its own under `branch/branch-`, which is where the load below and a - // read of the branch both read it, so a catalog answering for no branch schema does not fail a - // commit to one. - PAIMON_ASSIGN_OR_RAISE( - std::optional catalog_table_schema, - CatalogMainBranchSchemaJson(ctx.GetCatalog(), catalog_identifier, branch, "commit")); - // A format table commits by renaming files into place, so it never reaches the snapshot path - // below. The managed path here reads the commit's branch, so this reads the same one: the two - // must not dispatch on different schemas. - return ManagedTableOf(tmp_options.GetFileSystem(), specific_fs, root_path, branch, - ctx.GetOptions(), catalog_table_schema, ctx.GetIdentifier(), - ctx.GetTableId().value_or("")); -} - -/// The options a context carries against the ones the table was loaded with. A table serves the -/// options it was loaded with, so an option the context contradicts is refused rather than -/// honoured - a caller that means to change them copies the table. A table a load hands back -/// carries the context's own options merged in, so it agrees with the context by construction; -/// the refusal belongs to a table the caller held already, whose load is over. A key the table -/// was not loaded with is no contradiction: it names a runtime setting - a scan mode, a batch -/// size - which is none of the table's business, and is served as given. -Status CheckOptionsAgreeWithTable(const TableSchema& table_schema, - const std::map& context_options) { - const std::map& table_options = table_schema.Options(); - for (const auto& [key, value] : context_options) { - auto table_option = table_options.find(key); - if (table_option != table_options.end() && table_option->second != value) { - // A value may be a credential, and the error travels into logs: it is redacted the - // way every surface that exposes options redacts them. - return Status::Invalid( - fmt::format("option '{}' is '{}' in the context but '{}' on the table", key, - SensitiveConfigUtils::RedactValue(key, value), - SensitiveConfigUtils::RedactValue(key, table_option->second))); - } - } - return Status::OK(); -} - -/// The checks every operator entry point opens with: a context, a memory pool and an executor -/// must all be there for the operator to be built at all. `kind` names the operator the first -/// message speaks of, which is all that sets the entry points' messages apart. -template -Status CheckOperatorContext(const ContextPointer& context, const char* kind) { - if (context == nullptr) { - return Status::Invalid(fmt::format("{} context is null pointer", kind)); - } - if (context->GetMemoryPool() == nullptr) { - return Status::Invalid("memory pool is null pointer"); - } - if (context->GetExecutor() == nullptr) { - return Status::Invalid("executor is null pointer"); - } - return Status::OK(); -} - -} // namespace - -Table::Table(const std::shared_ptr& file_system, const std::string& root_path, - const std::shared_ptr& schema, const Identifier& identifier, - const std::string& uuid) - : file_system_(file_system), - root_path_(root_path), - schema_(schema), - identifier_(identifier), - uuid_(uuid) {} - FileStoreTable::FileStoreTable(const std::shared_ptr& store, const Identifier& identifier, const std::string& uuid) - : Table(store->file_system(), store->root_path(), store->schema(), identifier, uuid), + : Table(store->GetFileSystem(), store->GetRootPath(), store->GetSchema(), identifier, uuid), store_(store) {} Result> FileStoreTable::Create( @@ -580,12 +106,12 @@ Result> FileStoreTable::Create( const std::shared_ptr& schema, const Identifier& identifier, const std::string& uuid, const std::string& branch, const std::shared_ptr& schema_manager, const std::shared_ptr& snapshot_manager, - const std::map& fs_scheme_map) { - // The kind the schema's primary keys name, which no schema change can add or drop, so the - // kind the table is built as is the kind every schema its root path ever held describes. A - // caller without a schema gets the append-only kind, whose operators then report that no - // schema was found. A schema that is not a table schema - the read-only one a system table - // exposes, say - is none to serve a managed table through, so it is refused rather than cast. + const std::map& fs_scheme_map, + const std::map& dynamic_options) { + // The kind the schema's primary keys name, which no schema change can add or drop, so the kind + // the table is built as is the kind every schema its root path ever held describes. A caller + // without a schema gets the append-only kind, whose operators report that none was found; a + // schema that is not a table schema - a system table's read-only one - is refused, not cast. std::shared_ptr table_schema; if (schema != nullptr) { table_schema = std::dynamic_pointer_cast(schema); @@ -594,15 +120,18 @@ Result> FileStoreTable::Create( fmt::format("{} is not a managed table schema", identifier.ToString())); } } - // The store parses the schema's options, so a schema whose options do not parse is refused - // here: the table it names cannot be served, not by one operator more than another. + // The store parses the schema's options, so one that does not parse is refused here. PAIMON_ASSIGN_OR_RAISE(std::shared_ptr store, FileStore::Create(file_system, root_path, table_schema, branch, schema_manager, snapshot_manager, fs_scheme_map)); + std::shared_ptr table; if (table_schema != nullptr && !table_schema->PrimaryKeys().empty()) { - return std::make_shared(store, identifier, uuid); + table = std::make_shared(store, identifier, uuid); + } else { + table = std::make_shared(store, identifier, uuid); } - return std::make_shared(store, identifier, uuid); + table->dynamic_options_ = dynamic_options; + return table; } Result> FileStoreTable::Copy( @@ -612,69 +141,80 @@ Result> FileStoreTable::Copy( return Status::Invalid(fmt::format("not found latest schema of {}", FullName())); } // `type` is structural and read from the schema alone, the promise `Copy` makes: an option - // given at the call must not restyle the table. `branch` is the same kind of promise: the - // copy is the table it was copied from, on the branch that table serves, so a branch option - // given at the call moves neither - it would only claim a branch the copy does not serve - - // and a caller that means to move it has `SwitchToBranch`. Both are dropped before the - // merge, so the copy's options never speak of a branch its operators do not. + // given at the call must not restyle the table. `branch` is the same promise - the copy serves + // the branch this table does, and a caller that means to move has `SwitchToBranch`. Both are + // dropped before the merge, so the copy's options never speak of a branch it does not serve. std::map overridable = dynamic_options; overridable.erase(Options::TYPE); overridable.erase(Options::BRANCH); - // The physical layout the data on the path was written by is as immovable as the style or - // the branch: the copy serves the same data, so it is refused an option that would lay new - // data out differently than the data it serves was laid out. - PAIMON_RETURN_NOT_OK(CheckImmutableOptions(*loaded_schema, overridable)); + // The physical layout the data on the path was written by is as immovable as the style or the + // branch: the copy serves the same data. + PAIMON_RETURN_NOT_OK(SchemaValidation::CheckImmutableOptions(*loaded_schema, overridable)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr merged_schema, loaded_schema->MergeOptions(overridable)); - // The managers are handed over, not rebuilt, when the copy resolves the same file system - // this table does: its operators then read the schema cache this table already warmed. The - // file system is the one the caller opened this table through, not the one it resolved to: - // the copy resolves from its own merged options again, so a file-system option among the - // ones it was given speaks rather than being pinned to this table's resolution - and when - // one does change the resolution, fresh managers are built over it instead, so the metadata - // is read through the same backend the data is. The scheme map travels the same way, so the - // copy resolves custom schemes as this table does. + // The managers are handed over, not rebuilt, when the copy resolves the same file system this + // table does: its operators then read the schema cache this table already warmed. The copy + // resolves from the file system the caller opened this table through, not the one it resolved + // to, so a file-system option it was given speaks; when one does change the resolution, fresh + // managers are built over it instead. The scheme map travels the same way. const bool fs_resolution_may_change = - FileSystemResolutionMayChange(*loaded_schema, overridable); - return FileStoreTable::Create(store_->specified_file_system(), root_path_, + FileStore::FileSystemResolutionMayChange(*loaded_schema, overridable); + // The dynamic options the copy was given join the ones this table was loaded with - which a + // copy of a copy carries the same way - so a read's fallback reader serves what the copy's own + // reader does however many copies deep. The branch was dropped above, so none accumulates, and + // an option given again replaces the one carried, as the merge above does. + std::map accumulated = dynamic_options_; + for (const auto& [key, value] : overridable) { + accumulated.insert_or_assign(key, value); + } + return FileStoreTable::Create(store_->GetSpecifiedFileSystem(), root_path_, std::shared_ptr(std::move(merged_schema)), identifier_, CatalogUuid().value_or(""), CurrentBranch(), fs_resolution_may_change ? nullptr : GetSchemaManager(), fs_resolution_may_change ? nullptr : GetSnapshotManager(), - store_->fs_scheme_map()); + store_->GetFsSchemeMap(), accumulated); } Result> FileStoreTable::SwitchToBranch(const std::string& branch) const { const std::string target_branch = BranchManager::NormalizeBranch(branch); if (target_branch == CurrentBranch()) { - // The table already serves this branch: a copy of it - schema and managers shared - is - // the same table to the caller. - return FileStoreTable::Create(store_->specified_file_system(), root_path_, schema_, + // The table already serves this branch, so a copy of it is the same table to the caller. + return FileStoreTable::Create(store_->GetSpecifiedFileSystem(), root_path_, schema_, identifier_, CatalogUuid().value_or(""), CurrentBranch(), GetSchemaManager(), GetSnapshotManager(), - store_->fs_scheme_map()); + store_->GetFsSchemeMap(), dynamic_options_); } - // The schema of the branch switched to, read from its own directory, which a branch without - // one does not have. + // The schema of the branch switched to, read from its own directory. auto schema_manager = std::make_shared(file_system_, root_path_, target_branch); PAIMON_ASSIGN_OR_RAISE(std::optional> latest, schema_manager->Latest()); if (!latest) { return Status::Invalid(fmt::format("branch {} does not exist", target_branch)); } - // A branch other than main is named in the options the schema carries - the way a read - // context names it - so what reads through them serves the branch the managers point at. + // A branch other than main is named in the options the schema carries - the way a read context + // names it - so what reads through them serves the branch the managers point at. std::map overridable; if (target_branch != BranchManager::DEFAULT_MAIN_BRANCH) { overridable[Options::BRANCH] = target_branch; } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr branch_schema, (*latest)->MergeOptions(overridable)); - return FileStoreTable::Create(store_->specified_file_system(), root_path_, + // The branch's schema may name another file system than this table's options resolved - a + // recreation that changed the access configuration - and a manager kept through that + // disagreement would read the branch's metadata through the backend this table left, so the + // branch builds its own. The schema is in hand already, so only the cache is lost. + const std::shared_ptr branch_schema_manager = + (store_->GetSpecifiedFileSystem() == nullptr && + (!store_->GetFsSchemeMap().empty() || + !FileStore::FileSystemResolutionAgrees( + branch_schema->Options(), std::static_pointer_cast(schema_)->Options()))) + ? nullptr + : schema_manager; + return FileStoreTable::Create(store_->GetSpecifiedFileSystem(), root_path_, std::shared_ptr(std::move(branch_schema)), identifier_, CatalogUuid().value_or(""), target_branch, - schema_manager, /*snapshot_manager=*/nullptr, - store_->fs_scheme_map()); + branch_schema_manager, /*snapshot_manager=*/nullptr, + store_->GetFsSchemeMap()); } Status FileStoreTable::RestoreRealtimeCommittedProgress( @@ -695,6 +235,22 @@ Status FileStoreTable::RestoreRealtimeCommittedProgress( return Status::OK(); } +Status FileStoreTable::CheckOptionsAgreeWithTable( + const TableSchema& table_schema, const std::map& context_options) { + const std::map& table_options = table_schema.Options(); + for (const auto& [key, value] : context_options) { + auto table_option = table_options.find(key); + if (table_option != table_options.end() && table_option->second != value) { + // A value may be a credential and the error travels into logs, so it is redacted. + return Status::Invalid( + fmt::format("option '{}' is '{}' in the context but '{}' on the table", key, + SensitiveConfigUtils::RedactValue(key, value), + SensitiveConfigUtils::RedactValue(key, table_option->second))); + } + } + return Status::OK(); +} + std::unique_ptr FileStoreTable::WithTablePath(std::unique_ptr&& scan, const std::string& root_path) { scan->WithTablePath(root_path); @@ -703,32 +259,24 @@ std::unique_ptr FileStoreTable::WithTablePath(std::unique_ptr> FileStoreTable::NewWrite( std::unique_ptr ctx) const { - PAIMON_RETURN_NOT_OK(CheckOperatorContext(ctx, "write")); + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(ctx, "write")); PAIMON_RETURN_NOT_OK(CheckVersionManagementImplemented(ctx->GetCatalog())); std::shared_ptr specific_fs = ctx->GetSpecificFileSystem(); - // A catalog write names its table; the file system itself comes from the table, which its - // catalog's credentials already opened. + // A catalog write names its table; the file system comes from the table's own credentials. if (specific_fs == nullptr && ctx->GetFileSystemSchemeToIdentifierMap().empty() && ctx->GetCatalog() != nullptr && !ctx->GetIdentifier()) { return Status::Invalid("a catalog write requires a table identifier"); } - // The write goes to the branch the table serves: a table is the branch it was loaded for, - // so the context's own branch is not consulted. + // The write goes to the branch the table serves, not the context's own. PAIMON_ASSIGN_OR_RAISE(std::optional catalog_identifier, - MakeBranchCatalogIdentifier(ctx->GetIdentifier(), CurrentBranch())); - // The schema the table was constructed with, so a write serves the table as it was loaded - // rather than reading the schema again. - const std::shared_ptr& schema = store_->schema(); + CatalogUtils::BranchIdentifier(ctx->GetIdentifier(), CurrentBranch())); + const std::shared_ptr& schema = store_->GetSchema(); if (schema == nullptr) { return Status::Invalid( fmt::format("not found latest schema in branch {}", CurrentBranch())); } - // The options the store was built with are the ones this write serves: an option the - // context contradicts is refused rather than honoured, and a key the table was not loaded - // with names a runtime setting the write does not take - `Copy` derives the table that one - // belongs to. PAIMON_RETURN_NOT_OK(CheckOptionsAgreeWithTable(*schema, ctx->GetOptions())); - CoreOptions options = store_->options(); + CoreOptions options = store_->GetOptions(); auto arrow_schema = DataField::ConvertDataFieldsToArrowSchema(schema->Fields()); PAIMON_RETURN_NOT_OK(SchemaValidation::ValidateRealtime(*schema, options)); PAIMON_RETURN_NOT_OK(SchemaValidation::ValidateVectorStore(*schema, options)); @@ -767,64 +315,15 @@ Result> FileStoreTable::NewWrite( } Result> FileStoreWrite::Create(std::unique_ptr ctx) { - PAIMON_RETURN_NOT_OK(CheckOperatorContext(ctx, "write")); - // First the table the context names, then the context handed to that table's `NewWrite`. - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, TableOf(*ctx)); + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(ctx, "write")); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, FileStoreTableFactory::Create(*ctx)); return table->NewWrite(std::move(ctx)); } -namespace { - -Status ValidateRealtimeScan(const TableSchema& table_schema, const CoreOptions& core_options, - const ScanContext& context, bool read_optimized) { - if (!context.GetRealtimeContext()) { - return Status::OK(); - } - if (!core_options.RealtimeEnabled()) { - return Status::Invalid("real-time scan requires realtime.enabled=true"); - } - if (core_options.GetBucket() <= 0) { - return Status::Invalid("real-time union read requires fixed bucket mode"); - } - if (core_options.DataEvolutionEnabled()) { - return Status::Invalid("real-time union read does not support data evolution"); - } - if (!table_schema.PrimaryKeys().empty()) { - if (read_optimized) { - return Status::NotImplemented( - "PK real-time union read does not support read-optimized scans"); - } - PAIMON_RETURN_NOT_OK( - PrimaryKeyTableUtils::ValidateRealtimeOptions(core_options, table_schema)); - } - if (context.IsStreamingMode()) { - return Status::Invalid("real-time union read currently supports batch scans only"); - } - if (context.GetLimit()) { - return Status::Invalid("real-time union read does not support scan limit pushdown"); - } - if (context.GetGlobalIndexResult()) { - return Status::Invalid("real-time union read does not support global index splits"); - } - StartupMode startup_mode = core_options.GetStartupMode(); - const bool specified_snapshot = - startup_mode == StartupMode::FromSnapshot() && core_options.GetScanSnapshotId().has_value(); - if (!(startup_mode == StartupMode::LatestFull() || startup_mode == StartupMode::Latest() || - specified_snapshot)) { - return Status::Invalid( - "real-time union read requires the latest snapshot or an explicit snapshot id"); - } - return Status::OK(); -} - -} // namespace - Result> FileStoreTable::NewScan( std::unique_ptr context) const { - PAIMON_RETURN_NOT_OK(CheckOperatorContext(context, "scan")); - // The schema the table was constructed with, so a scan serves the table as it was loaded - // rather than reading the schema again. - const std::shared_ptr& table_schema = store_->schema(); + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "scan")); + const std::shared_ptr& table_schema = store_->GetSchema(); if (table_schema == nullptr) { return Status::Invalid("not found latest schema"); } @@ -832,20 +331,15 @@ Result> FileStoreTable::NewScan( PAIMON_ASSIGN_OR_RAISE(bool read_optimized, OptionsUtils::GetValueFromMap(context->GetOptions(), kReadOptimizedScanOption, false)); - // The options the store was built with are the ones this scan serves: an option the context - // contradicts is refused above, and a key the table was not loaded with names a runtime - // setting the scan does not take - `Copy` derives the table that one belongs to. The branch - // in them is the table's, not the context's: a table is the branch it was loaded for, so a - // branch the context carries is not honoured. - CoreOptions core_options = store_->options(); - // The cache is a runtime setting of this scan, not of the table: the copy the scan serves - // carries it. + // The branch in the store's options is the table's, so one the context carries is not honoured. + CoreOptions core_options = store_->GetOptions(); + // The cache is a runtime setting of this scan, not of the table. core_options.WithCache(context->GetCache()); PAIMON_RETURN_NOT_OK(SchemaValidation::ValidateRealtime(*table_schema, core_options)); PAIMON_RETURN_NOT_OK( - ValidateRealtimeScan(*table_schema, core_options, *context, read_optimized)); + RealtimeTableScan::ValidateScan(*table_schema, core_options, *context, read_optimized)); // validate options if (core_options.GetBucket() == -1) { if (!table_schema->PrimaryKeys().empty()) { @@ -880,7 +374,6 @@ Result> FileStoreTable::NewScan( store_->NewManifestFile(core_options, path_factory, partition_schema, context->GetMemoryPool())); - // The managers the table carries, whose schema cache the dispatch already warmed. PAIMON_ASSIGN_OR_RAISE( std::shared_ptr file_store_scan, NewFileStoreScan(manifest_list, manifest_file, arrow_schema, core_options, @@ -930,62 +423,11 @@ Result> FileStoreTable::NewScan( } Result> TableScan::Create(std::unique_ptr context) { - PAIMON_RETURN_NOT_OK(CheckOperatorContext(context, "scan")); - // First the table the context names, then the context handed to that table's `NewScan`. - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, TableOf(*context)); + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "scan")); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, FileStoreTableFactory::Create(*context)); return table->NewScan(std::move(context)); } -namespace { - -/// The read context of a fallback branch, whose schema is not the table's own: it is read -/// through a manager of that branch, and the options the read serves are the ones that schema -/// was loaded with - the same way the table's own read serves the store's - with the branch it -/// was read from in them. A branch the context carries is not honoured here. -/// @param file_system The file system the table was opened with, which the fallback reads -/// through in place of one resolved from the context. -/// @param root_path The root path the table was opened with, which the fallback reads from in -/// place of the one named by the context. -Result> CreateFallbackReadContext( - const std::shared_ptr& context, const std::string& branch, - const std::shared_ptr& file_system, const std::string& root_path) { - std::shared_ptr table_schema; - const auto& specific_table_schema = context->GetSpecificTableSchema(); - if (branch == BranchManager::DEFAULT_MAIN_BRANCH && specific_table_schema) { - PAIMON_ASSIGN_OR_RAISE(table_schema, - TableSchema::CreateFromJson(specific_table_schema.value())); - } else { - auto schema_manager = std::make_shared(file_system, root_path, branch); - PAIMON_ASSIGN_OR_RAISE(std::optional> latest_schema, - schema_manager->Latest()); - if (!latest_schema) { - return Status::Invalid( - fmt::format("schema file not found in path {}, branch {}", root_path, branch)); - } - table_schema = latest_schema.value(); - } - assert(table_schema); - auto options = table_schema->Options(); - // The dynamic options the load merged into the table this read serves - a sequence-number - // flag, a batch size - speak for the read, and the fallback branch serves the same read: - // they are merged over the fallback branch's own schema options the way the load merged - // them over the table's, so the fallback reader recognises what the main one does. `type` - // stays the branch's own: a copy never restyles the table, and neither does a fallback. - for (const auto& [key, value] : context->GetOptions()) { - if (key != Options::TYPE) { - options[key] = value; - } - } - if (branch != BranchManager::DEFAULT_MAIN_BRANCH) { - options[Options::BRANCH] = branch; - } else { - options.erase(Options::BRANCH); - } - return InternalReadContext::Create(context, table_schema, options, file_system, &root_path); -} - -} // namespace - Result> FileStoreTable::CreateTableRead( const std::shared_ptr& internal_context, const std::shared_ptr& memory_pool, @@ -1007,29 +449,29 @@ Result> FileStoreTable::CreateTableRead( std::shared_ptr path_factory, store_->NewPathFactory(arrow_schema, core_options, core_options.GetFileFormat()->Identifier(), memory_pool)); - return NewTableRead(path_factory, internal_context, memory_pool, executor); + // The reader is selected by the schema the read serves, not by the table's kind: an + // append-only table can have a primary-key fallback branch and vice versa, so a reader typed + // after the main table would fail the fallback's splits. + if (internal_context->GetPrimaryKeys().empty()) { + return std::make_unique(path_factory, internal_context, memory_pool, + executor); + } + return KeyValueTableRead::Create(path_factory, internal_context, memory_pool, executor); } Result> FileStoreTable::NewRead( std::unique_ptr context) const { std::shared_ptr read_context = std::move(context); - PAIMON_RETURN_NOT_OK(CheckOperatorContext(read_context, "read")); - // The schema the table was constructed with, so a read serves the table as it was loaded - // rather than reading the schema again. Its options are the ones the read serves: an option - // the context would have changed is refused rather than honoured, and a key the table was - // not loaded with names a runtime setting the read does not take - `Copy` derives the table - // that one belongs to. - const std::shared_ptr& loaded_schema = store_->schema(); + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(read_context, "read")); + const std::shared_ptr& loaded_schema = store_->GetSchema(); if (loaded_schema == nullptr) { return Status::Invalid("not found latest schema"); } PAIMON_RETURN_NOT_OK(CheckOptionsAgreeWithTable(*loaded_schema, read_context->GetOptions())); - // The options the store was built with, which the read reads through on the branch the - // table serves: a table is the branch it was loaded for, so the context's own branch is - // not consulted. - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr internal_context, - InternalReadContext::Create(read_context, loaded_schema, store_->options(), &root_path_)); + // The read reads through the branch the table serves, not the context's own. + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr internal_context, + InternalReadContext::Create(read_context, loaded_schema, + store_->GetOptions(), &root_path_)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_read, CreateTableRead(internal_context, read_context->GetMemoryPool(), @@ -1042,12 +484,11 @@ Result> FileStoreTable::NewRead( return std::move(table_read); } - // The fallback branch is not the table's, so its schema is read through a manager of its - // own rather than the table's. - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr fallback_context, - CreateFallbackReadContext(read_context, /*branch=*/scan_fallback_branch.value(), - file_system_, root_path_)); + // The fallback branch is not the table's, so its schema is read through its own manager. + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr fallback_context, + InternalReadContext::CreateForFallbackBranch( + read_context, /*branch=*/scan_fallback_branch.value(), file_system_, + root_path_, dynamic_options_)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr fallback_table_read, CreateTableRead(fallback_context, read_context->GetMemoryPool(), @@ -1058,62 +499,23 @@ Result> FileStoreTable::NewRead( } Result> TableRead::Create(std::unique_ptr ctx) { - PAIMON_RETURN_NOT_OK(CheckOperatorContext(ctx, "read")); - // First the table the context names, then the context handed to that table's `NewRead`. - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, TableOf(*ctx)); + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(ctx, "read")); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, FileStoreTableFactory::Create(*ctx)); return table->NewRead(std::move(ctx)); } -namespace { - -bool IsCatalogCommit(const CommitContext& ctx) { - return AsVersionManaged(ctx.GetCatalog()) != nullptr; -} - -Result> NewSnapshotCommit( - const CommitContext& ctx, const std::optional& catalog_identifier, - const std::shared_ptr& fs, - const std::shared_ptr& snapshot_manager) { - if (IsCatalogCommit(ctx)) { - if (!catalog_identifier) { - return Status::Invalid("a catalog commit requires a table identifier"); - } - // Reuse the caller's UUID so table recreation cannot redirect this commit. - return std::shared_ptr(std::make_shared( - ctx.GetCatalog(), catalog_identifier.value(), ctx.GetTableId())); - } - if (ctx.UseRESTCatalogCommit()) { - return std::shared_ptr( - std::make_shared(ctx.GetTableId())); - } - // Catalogs without version management may still supply a table ID. - if (ctx.GetTableId() && ctx.GetCatalog() == nullptr) { - return Status::Invalid( - "a commit which publishes the snapshot by writing it to the table directory sends no " - "commit table request, so there is nowhere to put a table id; commit through a " - "catalog with WithCatalog() or ask for the request with UseRESTCatalogCommit()"); - } - return std::shared_ptr( - std::make_shared(fs, snapshot_manager)); -} - -} // namespace - Result> FileStoreTable::NewCommit( std::unique_ptr ctx) const { - PAIMON_RETURN_NOT_OK(CheckOperatorContext(ctx, "commit")); + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(ctx, "commit")); PAIMON_RETURN_NOT_OK(CheckVersionManagementImplemented(ctx->GetCatalog())); std::shared_ptr specific_fs = ctx->GetSpecificFileSystem(); - // A catalog commit names its table; the file system itself comes from the table, which its - // catalog's credentials already opened. + // A catalog commit names its table; the file system comes from the table's own credentials. if (specific_fs == nullptr && ctx->GetCatalog() != nullptr && !ctx->GetIdentifier()) { return Status::Invalid("a catalog commit requires a table identifier"); } - // A commit commits to the root path the table was opened with and the branch the table - // serves, in place of the ones named by the context: a table is the branch it was loaded - // for, so the context's own branch is not consulted. + // A commit commits to the root path and branch the table was loaded with, not the context's. PAIMON_ASSIGN_OR_RAISE(std::optional catalog_identifier, - MakeBranchCatalogIdentifier(ctx->GetIdentifier(), CurrentBranch())); + CatalogUtils::BranchIdentifier(ctx->GetIdentifier(), CurrentBranch())); // The id a snapshot records the catalog's current schema by, read through the loader below // rather than trusted from a schema this commit will not re-read. FileStoreCommitImpl::SchemaIdLoader schema_id_loader; @@ -1135,21 +537,15 @@ Result> FileStoreTable::NewCommit( return data_schema->Id(); }; } - // The schema the table was constructed with, so a commit serves the table as it was loaded - // rather than reading the schema again. - const std::shared_ptr& schema = store_->schema(); + const std::shared_ptr& schema = store_->GetSchema(); if (schema == nullptr) { return Status::Invalid( fmt::format("not found latest schema in branch {}", CurrentBranch())); } std::shared_ptr arrow_schema = DataField::ConvertDataFieldsToArrowSchema(schema->Fields()); - // The options the store was built with are the ones this commit serves: an option the - // context contradicts is refused rather than honoured, and a key the table was not loaded - // with names a runtime setting the commit does not take - `Copy` derives the table that one - // belongs to. PAIMON_RETURN_NOT_OK(CheckOptionsAgreeWithTable(*schema, ctx->GetOptions())); - CoreOptions options = store_->options(); + CoreOptions options = store_->GetOptions(); PAIMON_RETURN_NOT_OK(SchemaValidation::ValidateRealtime(*schema, options)); assert(options.GetFileSystem()); assert(options.GetFileFormat()); @@ -1157,7 +553,8 @@ Result> FileStoreTable::NewCommit( PAIMON_ASSIGN_OR_RAISE(bool is_object_store, FileSystem::IsObjectStore(root_path_)); // Catalog commits do not require atomic rename on the object store. The inte-test marker is // not a table option, so it is read from the context the commit was handed. - if (is_object_store && !ctx->UseRESTCatalogCommit() && !IsCatalogCommit(*ctx) && + if (is_object_store && !ctx->UseRESTCatalogCommit() && + AsVersionManaged(ctx->GetCatalog()) == nullptr && ctx->GetOptions().find("enable-object-store-commit-in-inte-test") == ctx->GetOptions().end()) { return Status::NotImplemented( @@ -1195,9 +592,10 @@ Result> FileStoreTable::NewCommit( NewCommitScanSupplier(snapshot_manager, manifest_list, manifest_file, arrow_schema, options, ctx->GetExecutor(), ctx->GetMemoryPool()); - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr snapshot_commit, - NewSnapshotCommit(*ctx, catalog_identifier, options.GetFileSystem(), snapshot_manager)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr snapshot_commit, + SnapshotCommit::Create(ctx->GetCatalog(), catalog_identifier, + ctx->GetTableId(), ctx->UseRESTCatalogCommit(), + options.GetFileSystem(), snapshot_manager)); return std::make_unique( ctx->GetMemoryPool(), ctx->GetExecutor(), arrow_schema, root_path_, ctx->GetCommitUser(), @@ -1209,9 +607,8 @@ Result> FileStoreTable::NewCommit( Result> FileStoreCommit::Create( std::unique_ptr ctx) { - PAIMON_RETURN_NOT_OK(CheckOperatorContext(ctx, "commit")); - // First the table the context names, then the context handed to that table's `NewCommit`. - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, TableOf(*ctx)); + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(ctx, "commit")); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, FileStoreTableFactory::Create(*ctx)); return table->NewCommit(std::move(ctx)); } diff --git a/src/paimon/core/table/file_store_table.h b/src/paimon/core/table/file_store_table.h index df841d548..1fb307495 100644 --- a/src/paimon/core/table/file_store_table.h +++ b/src/paimon/core/table/file_store_table.h @@ -53,34 +53,25 @@ class SplitGenerator; /// decide the kind of managed table it is - `AppendOnlyFileStoreTable` or /// `PrimaryKeyFileStoreTable`, the `Create` factory below - and that kind builds the pieces the /// flows hand down: the write itself, the scans, the read and the split generator. -/// -/// Its `NewScan`/`NewRead`/`NewWrite`/`NewCommit` build the operators here, from the context plus -/// the state the table was constructed with, so they serve the table as it was loaded. The -/// context-only entry points - `TableScan::Create()`, `TableRead::Create()`, -/// `FileStoreWrite::Create()` and `FileStoreCommit::Create()` - load the table the context names -/// and hand the context to that table's `NewXxx`, which is how a context built without a table -/// reaches the same operators. class FileStoreTable : public Table { public: /// The managed table the schema describes, built as the kind its primary keys name: an /// `AppendOnlyFileStoreTable` when there are none, a `PrimaryKeyFileStoreTable` when there /// are, the way Java Paimon's `FileStoreTableFactory` does. Primary keys cannot be added or - /// dropped by a schema change, so the kind the table was built as is the kind every schema - /// its root path ever held describes. + /// dropped by a schema change, so the kind is the same for every schema the root path held. /// @param schema The schema the table was loaded with, whose primary keys name the kind and - /// whose options the operators serve. It is null when the caller had no schema, and - /// the table is built as the append-only kind - its operators then report that no - /// schema was found. + /// whose options the operators serve. Null when the caller had none, in which case the + /// table is built as the append-only kind and its operators report that none was found. /// @param branch The branch the table serves, which the managers below read and write on. - /// @param schema_manager A schema manager to carry instead of building one - the one a - /// context dispatch read the schema through, whose cache the operators then serve. - /// @param snapshot_manager A snapshot manager to carry instead of building one, the way - /// `Copy` hands its table's over so both share one. - /// A null manager is built over `root_path` on `branch`, so the operators always read - /// through the managers the table carries. + /// @param schema_manager A schema manager to carry instead of building one - see + /// `FileStore::Create`. + /// @param snapshot_manager A snapshot manager to carry instead of building one, the way `Copy` + /// hands its table's over so both share one - see `FileStore::Create`. /// @param fs_scheme_map The scheme-to-identifier map the caller's contexts resolve custom - /// schemes through, which the store's options resolve through as well - see - /// `FileStore::Create`. `Copy` hands its table's map over. + /// schemes through, which the store's options resolve through as well. `Copy` hands its + /// table's map over. + /// @param dynamic_options The options a copy was given over the ones its schema stored, which + /// `Copy` accumulates - see the member of the same name. static Result> Create( const std::shared_ptr& file_system, const std::string& root_path, const std::shared_ptr& schema, const Identifier& identifier, @@ -88,26 +79,19 @@ class FileStoreTable : public Table { const std::string& branch = BranchManager::DEFAULT_MAIN_BRANCH, const std::shared_ptr& schema_manager = nullptr, const std::shared_ptr& snapshot_manager = nullptr, - const std::map& fs_scheme_map = {}); + const std::map& fs_scheme_map = {}, + const std::map& dynamic_options = {}); - /// The branch this table serves, as resolved when the table was built: the schema the table - /// was loaded with came from it, and its managers point at it. const std::string& CurrentBranch() const { - return store_->branch(); + return store_->CurrentBranch(); } - /// The schema manager the operators read through. It carries the schema cache of `root_path` - /// on `CurrentBranch()`, which every operator serves rather than each reading the schema - /// files again. const std::shared_ptr& GetSchemaManager() const { - return store_->schema_manager(); + return store_->GetSchemaManager(); } - /// The snapshot manager a scan reads through. A write or a commit builds its own: it arms - /// the manager with a catalog snapshot loader, which a shared one would keep serving to - /// scans afterwards. const std::shared_ptr& GetSnapshotManager() const { - return store_->snapshot_manager(); + return store_->GetSnapshotManager(); } std::string Name() const override { @@ -118,17 +102,15 @@ class FileStoreTable : public Table { return identifier_.GetFullName(); } - /// Copies the table with `dynamic_options` on top of the ones its schema stored, the - /// precedence `Copy` promises. The file system, root path and identifier stay as they are: - /// they name this table, which an option does not rename. + /// The file system, root path and identifier stay as they are: they name this table, which an + /// option does not rename. Result> Copy( const std::map& dynamic_options) const override; /// Returns the table of `branch`, as Java Paimon's `switchToBranch` does. The branch switched - /// to is served as it is: its schema is read from its own directory, so the dynamic options - /// of this table are not carried over. Switching to the branch already served hands back a - /// copy of this table, schema and managers shared, and a branch with no schema does not - /// exist. + /// to is served as it is: its schema is read from its own directory, so this table's dynamic + /// options are not carried over. Switching to the branch already served hands back a copy of + /// this table, schema and managers shared, and a branch with no schema does not exist. Result> SwitchToBranch(const std::string& branch) const; Result> NewScan(std::unique_ptr context) const override; @@ -142,38 +124,43 @@ class FileStoreTable : public Table { std::unique_ptr context) const override; private: - /// The read `NewRead` hands back, from the internal context it read the schema into: the - /// path factory comes from the store - a fallback branch's schema carries the table's - /// partition keys, which no schema change can alter - and the read itself is the kind's, - /// `NewTableRead`. + /// The options a context carries against the ones the table was loaded with. A table serves + /// the options it was loaded with, so an option the context contradicts is refused rather than + /// honoured - a caller that means to change them copies the table. A table a load hands back + /// carries the context's own options merged in - `FileStoreTableFactory` does the merging - so + /// it agrees by construction; the refusal belongs to a table the caller held already, whose + /// load is over. A key the table was not loaded with is no contradiction: it names a runtime + /// setting - a scan mode, a batch size - which is none of the table's business, and is served + /// as given. + static Status CheckOptionsAgreeWithTable( + const TableSchema& table_schema, const std::map& context_options); + + /// The read `NewRead` hands back, from the internal context it read the schema into: the path + /// factory comes from the store, and the reader is selected by the schema the read serves - the + /// primary keys it carries - rather than by the table's own kind. Result> CreateTableRead( const std::shared_ptr& internal_context, const std::shared_ptr& memory_pool, const std::shared_ptr& executor) const; protected: - /// The base constructor the kinds of managed table build through, `Create` above: the store - /// the table was loaded with carries every state the kinds hand down - see `Create` for what - /// each piece of it is. + /// The base constructor the kinds of managed table build through: the store carries every state + /// the kinds hand down - see `Create` for what each piece of it is. FileStoreTable(const std::shared_ptr& store, const Identifier& identifier, const std::string& uuid); /// The committed progress a real-time write restores, from the latest snapshot the snapshot - /// manager reads: where the write picks up after, which both kinds of real-time write ask - /// for the same way. + /// manager reads: where the write picks up after, which both kinds ask for the same way. static Status RestoreRealtimeCommittedProgress( const std::shared_ptr& realtime_context, const std::shared_ptr& snapshot_manager, const CoreOptions& options); - /// The scan the table's root path names, which the kinds of file store scan read from: the - /// kinds hand their scans back through it. + /// The scan the table's root path names, which the kinds of file store scan read from. static std::unique_ptr WithTablePath(std::unique_ptr&& scan, const std::string& root_path); - /// The write of this kind of table, which `NewWrite` hands the shared flow's pieces to: the - /// kind's writer, built over the path factory, snapshot manager and options that flow - /// resolved. The schema is the store's own, the one the table was loaded with, and the - /// branch the write serves is the table's, `CurrentBranch()`. + /// The write of this kind of table, which `NewWrite` hands the shared flow's pieces to. The + /// schema and the branch the write serves are the table's own. /// @param io_manager The spill manager the context's temp directory names, or null when the /// context named none. /// @param ignore_previous_files Whether the write starts over rather than appending, which @@ -186,8 +173,7 @@ class FileStoreTable : public Table { const std::shared_ptr& io_manager, bool ignore_previous_files) const = 0; /// The scan of this kind of table, which `NewScan` hands the manifest list and file it built - /// to. The schema is the store's own and the snapshot and schema managers are the table's - /// own, which the scan serves. + /// to, over the table's own schema and managers. virtual Result> NewFileStoreScan( const std::shared_ptr& manifest_list, const std::shared_ptr& manifest_file, @@ -195,22 +181,12 @@ class FileStoreTable : public Table { const std::shared_ptr& executor, const std::shared_ptr& memory_pool, const ScanContext* context) const = 0; - /// The split generator of this kind of table, Java Paimon's `splitGenerator()`. The schema - /// it reads its kind's keys from is the store's own. + /// The split generator of this kind of table, Java Paimon's `splitGenerator()`. virtual Result> NewSplitGenerator( const CoreOptions& core_options) const = 0; - /// The read of this kind of table, which `NewRead` hands the internal context it read the - /// schema into and the path factory it built to. - virtual Result> NewTableRead( - const std::shared_ptr& path_factory, - const std::shared_ptr& internal_context, - const std::shared_ptr& memory_pool, - const std::shared_ptr& executor) const = 0; - - /// The manifest entry scan supplier of this kind of table, which `NewCommit` hands the - /// manifest list and file it built to. The schema is the store's own, and the schema manager - /// the scans it hands out read through is the table's own. + /// The manifest entry scan supplier of this kind of table, which `NewCommit` hands the manifest + /// list and file it built to. virtual CommitScanner::ScanSupplier NewCommitScanSupplier( const std::shared_ptr& snapshot_manager, const std::shared_ptr& manifest_list, @@ -219,9 +195,14 @@ class FileStoreTable : public Table { const std::shared_ptr& executor, const std::shared_ptr& pool) const = 0; - /// The storage this table serves: the file system, root path, schema, branch and managers it - /// was loaded with, and the factories that build the pieces every operator opens with. + /// The storage this table serves - see `FileStore`. std::shared_ptr store_; + + /// The dynamic options the table was loaded with over the ones its schema stored, which `Copy` + /// accumulates into the copy it derives. They stay separate from the schema they were merged + /// into because the fallback reader of a read needs them alone: it serves another branch's + /// schema, over which the same options are merged again. + std::map dynamic_options_; }; } // namespace paimon diff --git a/src/paimon/core/table/format/format_table.cpp b/src/paimon/core/table/format/format_table.cpp index cb4141dde..d4b611eac 100644 --- a/src/paimon/core/table/format/format_table.cpp +++ b/src/paimon/core/table/format/format_table.cpp @@ -39,6 +39,7 @@ #include "paimon/core/table/format/format_table_file_store_write.h" #include "paimon/core/table/format/format_table_read.h" #include "paimon/core/table/format/format_table_scan.h" +#include "paimon/core/utils/context_utils.h" #include "paimon/defs.h" #include "paimon/file_store_commit.h" #include "paimon/file_store_write.h" @@ -226,15 +227,7 @@ Result> FormatTable::GetArrowSchema() const { Result> FormatTable::NewScan( std::unique_ptr context) const { std::shared_ptr shared_context = std::move(context); - if (shared_context == nullptr) { - return Status::Invalid("scan context is null pointer"); - } - if (shared_context->GetMemoryPool() == nullptr) { - return Status::Invalid("memory pool is null pointer"); - } - if (shared_context->GetExecutor() == nullptr) { - return Status::Invalid("executor is null pointer"); - } + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(shared_context, "scan")); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table, CopySelf(shared_context->GetOptions())); // Anything a `ScanContext` carries that a format table cannot honour is refused rather than @@ -280,15 +273,7 @@ Result> FormatTable::NewScan( Result> FormatTable::NewRead( std::unique_ptr context) const { std::shared_ptr read_context = std::move(context); - if (read_context == nullptr) { - return Status::Invalid("read context is null pointer"); - } - if (read_context->GetMemoryPool() == nullptr) { - return Status::Invalid("memory pool is null pointer"); - } - if (read_context->GetExecutor() == nullptr) { - return Status::Invalid("executor is null pointer"); - } + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(read_context, "read")); if (read_context->HasVectorOrFullTextSearch()) { return Status::NotImplemented( "VectorSearch or FullTextSearch is not supported for format tables"); @@ -305,15 +290,7 @@ Result> FormatTable::NewRead( Result> FormatTable::NewWrite( std::unique_ptr context) const { - if (context == nullptr) { - return Status::Invalid("write context is null pointer"); - } - if (context->GetMemoryPool() == nullptr) { - return Status::Invalid("memory pool is null pointer"); - } - if (context->GetExecutor() == nullptr) { - return Status::Invalid("executor is null pointer"); - } + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "write")); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table, CopySelf(context->GetOptions())); if (context->GetCatalog() != nullptr) { return Status::Invalid( @@ -346,15 +323,7 @@ Result> FormatTable::NewWrite( Result> FormatTable::NewCommit( std::unique_ptr context) const { - if (context == nullptr) { - return Status::Invalid("commit context is null pointer"); - } - if (context->GetMemoryPool() == nullptr) { - return Status::Invalid("memory pool is null pointer"); - } - if (context->GetExecutor() == nullptr) { - return Status::Invalid("executor is null pointer"); - } + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "commit")); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr given_table, CopySelf(context->GetOptions())); if (!context->IgnoreEmptyCommit()) { diff --git a/src/paimon/core/table/primary_key_file_store_table.cpp b/src/paimon/core/table/primary_key_file_store_table.cpp index 06e27e0d1..9a5875c9f 100644 --- a/src/paimon/core/table/primary_key_file_store_table.cpp +++ b/src/paimon/core/table/primary_key_file_store_table.cpp @@ -52,7 +52,6 @@ #include "paimon/core/snapshot.h" #include "paimon/core/table/bucket_mode.h" #include "paimon/core/table/file_store.h" -#include "paimon/core/table/source/key_value_table_read.h" #include "paimon/core/table/source/merge_tree_split_generator.h" #include "paimon/core/table/source/split_generator.h" #include "paimon/core/utils/primary_key_table_utils.h" @@ -108,7 +107,7 @@ Result> PrimaryKeyFileStoreTable::CreateWrite( const std::shared_ptr& snapshot_manager, const std::shared_ptr& io_manager, bool ignore_previous_files) const { // pk table - const std::shared_ptr& schema = store_->schema(); + const std::shared_ptr& schema = store_->GetSchema(); if (ctx.GetRealtimeContext()) { PAIMON_RETURN_NOT_OK(PrimaryKeyTableUtils::ValidateRealtimeOptions(options, *schema)); if (ignore_previous_files) { @@ -228,7 +227,7 @@ Result> PrimaryKeyFileStoreTable::NewFileStoreSca const std::shared_ptr& executor, const std::shared_ptr& memory_pool, const ScanContext* context) const { // TODO(liancheng.lsz): support fallback branch in scan - const std::shared_ptr& table_schema = store_->schema(); + const std::shared_ptr& table_schema = store_->GetSchema(); std::unique_ptr scan; PAIMON_ASSIGN_OR_RAISE( scan, KeyValueFileStoreScan::Create(GetSnapshotManager(), GetSchemaManager(), manifest_list, @@ -243,7 +242,7 @@ Result> PrimaryKeyFileStoreTable::NewSplitGenera auto source_split_target_size = core_options.GetSourceSplitTargetSize(); auto source_split_open_file_cost = core_options.GetSourceSplitOpenFileCost(); // TODO(liancheng.lsz): support evolution - const std::shared_ptr& table_schema = store_->schema(); + const std::shared_ptr& table_schema = store_->GetSchema(); PAIMON_ASSIGN_OR_RAISE(std::vector trimmed_primary_keys, table_schema->TrimmedPrimaryKeys()); PAIMON_ASSIGN_OR_RAISE(std::vector trimmed_pk_fields, @@ -256,14 +255,6 @@ Result> PrimaryKeyFileStoreTable::NewSplitGenera core_options.DeletionVectorsEnabled(), core_options.GetMergeEngine(), key_comparator); } -Result> PrimaryKeyFileStoreTable::NewTableRead( - const std::shared_ptr& path_factory, - const std::shared_ptr& internal_context, - const std::shared_ptr& memory_pool, - const std::shared_ptr& executor) const { - return KeyValueTableRead::Create(path_factory, internal_context, memory_pool, executor); -} - CommitScanner::ScanSupplier PrimaryKeyFileStoreTable::NewCommitScanSupplier( const std::shared_ptr& snapshot_manager, const std::shared_ptr& manifest_list, @@ -273,7 +264,7 @@ CommitScanner::ScanSupplier PrimaryKeyFileStoreTable::NewCommitScanSupplier( // The schema manager the table carries, which the scans this supplier hands out read the // schema through. const std::shared_ptr& schema_manager = GetSchemaManager(); - const std::shared_ptr& table_schema = store_->schema(); + const std::shared_ptr& table_schema = store_->GetSchema(); return [snapshot_manager, schema_manager, manifest_list, manifest_file, table_schema, arrow_schema, options, executor, pool](const std::shared_ptr& scan_filter) -> Result> { diff --git a/src/paimon/core/table/primary_key_file_store_table.h b/src/paimon/core/table/primary_key_file_store_table.h index 9b0ed5c9a..ce7547a71 100644 --- a/src/paimon/core/table/primary_key_file_store_table.h +++ b/src/paimon/core/table/primary_key_file_store_table.h @@ -63,12 +63,6 @@ class PrimaryKeyFileStoreTable : public FileStoreTable { Result> NewSplitGenerator( const CoreOptions& core_options) const override; - Result> NewTableRead( - const std::shared_ptr& path_factory, - const std::shared_ptr& internal_context, - const std::shared_ptr& memory_pool, - const std::shared_ptr& executor) const override; - CommitScanner::ScanSupplier NewCommitScanSupplier( const std::shared_ptr& snapshot_manager, const std::shared_ptr& manifest_list, diff --git a/src/paimon/core/table/source/realtime_table_scan.cpp b/src/paimon/core/table/source/realtime_table_scan.cpp index 42a7171b3..cfb61c17d 100644 --- a/src/paimon/core/table/source/realtime_table_scan.cpp +++ b/src/paimon/core/table/source/realtime_table_scan.cpp @@ -27,17 +27,21 @@ #include "fmt/format.h" #include "paimon/common/utils/scope_guard.h" +#include "paimon/core/core_options.h" #include "paimon/core/operation/commit/realtime_commit_properties.h" +#include "paimon/core/schema/table_schema.h" #include "paimon/core/snapshot.h" #include "paimon/core/table/source/data_split_impl.h" #include "paimon/core/table/source/plan_impl.h" #include "paimon/core/table/source/realtime_split.h" #include "paimon/core/utils/file_store_path_factory.h" +#include "paimon/core/utils/primary_key_table_utils.h" #include "paimon/core/utils/snapshot_manager.h" #include "paimon/realtime/realtime_context.h" #include "paimon/realtime/realtime_store.h" #include "paimon/scan_context.h" #include "paimon/status.h" +#include "paimon/table/source/startup_mode.h" namespace paimon { @@ -203,6 +207,49 @@ Result>> RealtimeTableScan::CreateRealtimeSpl return result; } +Status RealtimeTableScan::ValidateScan(const TableSchema& table_schema, + const CoreOptions& core_options, const ScanContext& context, + bool read_optimized) { + if (!context.GetRealtimeContext()) { + return Status::OK(); + } + if (!core_options.RealtimeEnabled()) { + return Status::Invalid("real-time scan requires realtime.enabled=true"); + } + if (core_options.GetBucket() <= 0) { + return Status::Invalid("real-time union read requires fixed bucket mode"); + } + if (core_options.DataEvolutionEnabled()) { + return Status::Invalid("real-time union read does not support data evolution"); + } + if (!table_schema.PrimaryKeys().empty()) { + if (read_optimized) { + return Status::NotImplemented( + "PK real-time union read does not support read-optimized scans"); + } + PAIMON_RETURN_NOT_OK( + PrimaryKeyTableUtils::ValidateRealtimeOptions(core_options, table_schema)); + } + if (context.IsStreamingMode()) { + return Status::Invalid("real-time union read currently supports batch scans only"); + } + if (context.GetLimit()) { + return Status::Invalid("real-time union read does not support scan limit pushdown"); + } + if (context.GetGlobalIndexResult()) { + return Status::Invalid("real-time union read does not support global index splits"); + } + StartupMode startup_mode = core_options.GetStartupMode(); + const bool specified_snapshot = + startup_mode == StartupMode::FromSnapshot() && core_options.GetScanSnapshotId().has_value(); + if (!(startup_mode == StartupMode::LatestFull() || startup_mode == StartupMode::Latest() || + specified_snapshot)) { + return Status::Invalid( + "real-time union read requires the latest snapshot or an explicit snapshot id"); + } + return Status::OK(); +} + RealtimeTableScan::RealtimeTableScan(std::unique_ptr&& disk_scan, bool pk_table, const std::shared_ptr& realtime_context, const std::shared_ptr& path_factory, diff --git a/src/paimon/core/table/source/realtime_table_scan.h b/src/paimon/core/table/source/realtime_table_scan.h index 821e67fd7..318319139 100644 --- a/src/paimon/core/table/source/realtime_table_scan.h +++ b/src/paimon/core/table/source/realtime_table_scan.h @@ -30,14 +30,23 @@ namespace paimon { +class CoreOptions; class FileStorePathFactory; class FileSystem; +class ScanContext; class ScanFilter; class SnapshotManager; +class TableSchema; /// Adds process-local memory splits to a normal data-table batch scan. class RealtimeTableScan : public TableScan { public: + /// The checks a scan that asks for a real-time union read opens with, run before one of these + /// is built: the table has to be able to serve it, and the options have to name a scan one + /// can be served over. A context that asks for none of it passes straight through. + static Status ValidateScan(const TableSchema& table_schema, const CoreOptions& core_options, + const ScanContext& context, bool read_optimized); + RealtimeTableScan(std::unique_ptr&& disk_scan, bool pk_table, const std::shared_ptr& realtime_context, const std::shared_ptr& path_factory, diff --git a/src/paimon/core/table/system/audit_log_system_table.cpp b/src/paimon/core/table/system/audit_log_system_table.cpp index e191b653a..09fd6328f 100644 --- a/src/paimon/core/table/system/audit_log_system_table.cpp +++ b/src/paimon/core/table/system/audit_log_system_table.cpp @@ -44,6 +44,7 @@ #include "paimon/core/schema/table_schema.h" #include "paimon/core/table/source/data_split_impl.h" #include "paimon/core/table/source/key_value_table_read.h" +#include "paimon/core/utils/context_utils.h" #include "paimon/defs.h" #include "paimon/read_context.h" #include "paimon/scan_context.h" @@ -386,6 +387,7 @@ Result> AuditLogSystemTable::ArrowSchema() const Result> AuditLogSystemTable::NewScan( std::unique_ptr ctx) const { const std::shared_ptr context = std::move(ctx); + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "scan")); if (context->GetScanFilters() && context->GetScanFilters()->GetPredicate()) { return Status::NotImplemented("audit_log system table predicate pushdown is not supported"); } @@ -395,9 +397,7 @@ Result> AuditLogSystemTable::NewScan( .WithStreamingMode(context->IsStreamingMode()) .WithMemoryPool(context->GetMemoryPool()) .WithExecutor(context->GetExecutor()) - // The data table behind this view is served through the file system the view itself was - // opened with, so a context without one of its own falls back to it rather than the - // default backend. + // The view's own file system when the context names none, rather than the default backend. .WithFileSystem(context->GetSpecificFileSystem() != nullptr ? context->GetSpecificFileSystem() : GetFileSystem()) @@ -420,6 +420,8 @@ Result> AuditLogSystemTable::NewScan( Result> AuditLogSystemTable::NewRead( std::unique_ptr ctx) const { const std::shared_ptr context = std::move(ctx); + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "read")); + PAIMON_RETURN_NOT_OK(CheckNoSearch(context)); return NewChangelogRead(context, std::make_shared()); } @@ -458,10 +460,8 @@ Result> AuditLogSystemTable::CreateDataReadContext( PAIMON_ASSIGN_OR_RAISE(std::shared_ptr base_read_schema, BaseReadSchema()); PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, CoreOptions::FromMap(read_options)); ReadContextBuilder builder(table_path_); - // The data table behind this view is served through the file system the view itself was - // opened with, but only when the context names neither a file system of its own nor a - // scheme map to resolve through - a file system and a scheme map cannot both be named, and - // the context's own answer always wins. + // The view's own file system, but only when the context names neither one of its own nor a + // scheme map to resolve through - a file system and a scheme map cannot both be named. std::shared_ptr data_table_fs = context->GetSpecificFileSystem(); if (data_table_fs == nullptr && context->GetFileSystemSchemeToIdentifierMap().empty()) { data_table_fs = GetFileSystem(); diff --git a/src/paimon/core/table/system/binlog_system_table.cpp b/src/paimon/core/table/system/binlog_system_table.cpp index 8a821d45f..a3c39078d 100644 --- a/src/paimon/core/table/system/binlog_system_table.cpp +++ b/src/paimon/core/table/system/binlog_system_table.cpp @@ -31,6 +31,7 @@ #include "paimon/common/utils/arrow/status_utils.h" #include "paimon/core/core_options.h" #include "paimon/core/schema/table_schema.h" +#include "paimon/core/utils/context_utils.h" #include "paimon/defs.h" #include "paimon/read_context.h" #include "paimon/table/source/table_read.h" @@ -100,6 +101,8 @@ Result> BinlogSystemTable::ArrowSchema() const { Result> BinlogSystemTable::NewRead( std::unique_ptr ctx) const { const std::shared_ptr context = std::move(ctx); + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "read")); + PAIMON_RETURN_NOT_OK(CheckNoSearch(context)); return NewChangelogRead(context, CreateBinlogBatchConverter()); } diff --git a/src/paimon/core/table/system/global_system_tables.h b/src/paimon/core/table/system/global_system_tables.h index 3d6936262..eadc83af1 100644 --- a/src/paimon/core/table/system/global_system_tables.h +++ b/src/paimon/core/table/system/global_system_tables.h @@ -33,7 +33,14 @@ class FileSystem; /// Context passed to global system table constructors, providing catalog-level /// access for enumerating databases, tables, and reading metadata. struct GlobalSystemTableContext { - const Catalog* catalog = nullptr; // non-owning pointer + /// The catalog the table reads its rows through, non-owning: the rows are built lazily, + /// so a table that outlives the call that built it must be handed the shared ownership + /// below too, or this pointer dangles at the first `BuildRows`. + const Catalog* catalog = nullptr; + /// The shared ownership a returned table keeps of its catalog, so its lazy `BuildRows` + /// cannot read through one already destroyed. Empty where the table is transient - used + /// within the call, before the catalog can go away. + std::shared_ptr catalog_retainer; std::shared_ptr fs; std::string warehouse; std::map catalog_options; diff --git a/src/paimon/core/table/system/in_memory_system_table.cpp b/src/paimon/core/table/system/in_memory_system_table.cpp index 773f621b8..d21d6fb4d 100644 --- a/src/paimon/core/table/system/in_memory_system_table.cpp +++ b/src/paimon/core/table/system/in_memory_system_table.cpp @@ -29,6 +29,7 @@ #include "paimon/common/utils/checked_cast.h" #include "paimon/core/io/generic_row_to_arrow_array_converter.h" #include "paimon/core/table/system/system_table_scan.h" +#include "paimon/core/utils/context_utils.h" #include "paimon/memory/memory_pool.h" #include "paimon/read_context.h" #include "paimon/scan_context.h" @@ -117,6 +118,10 @@ Result> InMemorySystemTable::NewScan( Result> InMemorySystemTable::NewRead( std::unique_ptr ctx) const { const std::shared_ptr context = std::move(ctx); + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "read")); + // The rows this view serves are its own, read straight out of memory: there is no search + // over them, so a search the caller asked for would be quietly answered with plain rows. + PAIMON_RETURN_NOT_OK(CheckNoSearch(context)); return std::make_unique( checked_pointer_cast(shared_from_this()), context->GetMemoryPool()); diff --git a/src/paimon/core/table/system/read_optimized_system_table.cpp b/src/paimon/core/table/system/read_optimized_system_table.cpp index d7193b5cb..65a48c048 100644 --- a/src/paimon/core/table/system/read_optimized_system_table.cpp +++ b/src/paimon/core/table/system/read_optimized_system_table.cpp @@ -27,6 +27,7 @@ #include "paimon/common/types/data_field.h" #include "paimon/core/schema/table_schema.h" #include "paimon/core/table/source/read_optimized_scan_options.h" +#include "paimon/core/utils/context_utils.h" #include "paimon/defs.h" #include "paimon/read_context.h" #include "paimon/scan_context.h" @@ -61,6 +62,7 @@ std::map ReadOptimizedSystemTable::ReadOptimizedOption Result> ReadOptimizedSystemTable::NewScan( std::unique_ptr ctx) const { const std::shared_ptr context = std::move(ctx); + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "scan")); if (context->GetRealtimeContext() && !table_schema_->PrimaryKeys().empty()) { return Status::NotImplemented( "PK real-time union read does not support read-optimized scans"); @@ -71,9 +73,7 @@ Result> ReadOptimizedSystemTable::NewScan( .WithStreamingMode(context->IsStreamingMode()) .WithMemoryPool(context->GetMemoryPool()) .WithExecutor(context->GetExecutor()) - // The data table behind this view is served through the file system the view itself was - // opened with, so a context without one of its own falls back to it rather than the - // default backend. + // The view's own file system when the context names none, rather than the default backend. .WithFileSystem(context->GetSpecificFileSystem() != nullptr ? context->GetSpecificFileSystem() : GetFileSystem()) @@ -108,10 +108,8 @@ Result> ReadOptimizedSystemTable::CreateDataReadCon branch = branch_iter->second; } ReadContextBuilder builder(table_path_); - // The data table behind this view is served through the file system the view itself was - // opened with, but only when the context names neither a file system of its own nor a - // scheme map to resolve through - a file system and a scheme map cannot both be named, and - // the context's own answer always wins. + // The view's own file system, but only when the context names neither one of its own nor a + // scheme map to resolve through - a file system and a scheme map cannot both be named. std::shared_ptr data_table_fs = context->GetSpecificFileSystem(); if (data_table_fs == nullptr && context->GetFileSystemSchemeToIdentifierMap().empty()) { data_table_fs = GetFileSystem(); @@ -151,6 +149,8 @@ Result> ReadOptimizedSystemTable::CreateDataReadCon Result> ReadOptimizedSystemTable::NewRead( std::unique_ptr ctx) const { const std::shared_ptr context = std::move(ctx); + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "read")); + PAIMON_RETURN_NOT_OK(CheckNoSearch(context)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr base_context, CreateDataReadContext(context)); return TableRead::Create(std::move(base_context)); diff --git a/src/paimon/core/table/system/system_table.cpp b/src/paimon/core/table/system/system_table.cpp index cf4864071..0161ad6b7 100644 --- a/src/paimon/core/table/system/system_table.cpp +++ b/src/paimon/core/table/system/system_table.cpp @@ -42,6 +42,7 @@ #include "paimon/core/utils/branch_manager.h" #include "paimon/file_store_commit.h" #include "paimon/file_store_write.h" +#include "paimon/read_context.h" #include "paimon/status.h" #include "paimon/write_context.h" @@ -185,6 +186,14 @@ SystemTable::SystemTable(const std::shared_ptr& fs, const std::strin : Table(fs, table_path, /*schema=*/nullptr, Identifier(Identifier::kUnknownDatabase, "system_table"), /*uuid=*/"") {} +Status SystemTable::CheckNoSearch(const std::shared_ptr& context) { + if (context->HasVectorOrFullTextSearch()) { + return Status::NotImplemented( + "VectorSearch or FullTextSearch does not support system-table reads yet"); + } + return Status::OK(); +} + Status SystemTable::AdoptOwnSchema() { PAIMON_ASSIGN_OR_RAISE(std::shared_ptr arrow_schema, ArrowSchema()); schema_ = std::make_shared(std::move(arrow_schema)); diff --git a/src/paimon/core/table/system/system_table.h b/src/paimon/core/table/system/system_table.h index 96eead153..4c235d3cf 100644 --- a/src/paimon/core/table/system/system_table.h +++ b/src/paimon/core/table/system/system_table.h @@ -50,13 +50,10 @@ struct SystemTablePath { bool is_global = false; }; -/// Base class for table-scoped system tables such as `T$options` and `T$snapshots`. -/// -/// A system table is the read-only view a `T$system` identifier names: it exposes a schema of -/// its own and serves scans and reads from the data table it belongs to, through the same -/// `Table` interface as every other kind of table. `Name()` - which each kind of system table -/// answers for itself - is its full name as well: the identifier it is opened by belongs to the -/// data table underneath and would name that one. +/// Base class for table-scoped system tables such as `T$options` and `T$snapshots`: the read-only +/// view a `T$system` identifier names, which exposes a schema of its own and serves scans and +/// reads from the data table it belongs to. `Name()` - which each kind answers for itself - is +/// its full name as well: the identifier it is opened by belongs to the data table underneath. class SystemTable : public Table, public std::enable_shared_from_this { public: std::string FullName() const override { @@ -66,14 +63,11 @@ class SystemTable : public Table, public std::enable_shared_from_this> ArrowSchema() const = 0; - /// A system table has no options of its own to override: its schema is the read-only view it - /// exposes, which is not reconfigured by a copy. The data table underneath is copied through - /// its own `Table`. + /// A system table is read-only: copying it, writing it and committing for it are refused + /// rather than routed to the data table underneath, which its own `Table` serves. Result> Copy( const std::map& dynamic_options) const override; - /// A system table is read-only: writing it or committing for it is refused rather than - /// routed to the data table underneath. Result> NewWrite( std::unique_ptr context) const override; @@ -85,9 +79,13 @@ class SystemTable : public Table, public std::enable_shared_from_this& fs, const std::string& table_path); - /// Loaders call this once the object exists, because `ArrowSchema()` is virtual and cannot - /// run from a constructor: it fills the schema the base hands out with this table's own - /// read-only `SystemTableSchema`. + /// Refuses a read whose context asks for a search: each system table builds a context of its + /// own for the data table underneath, which carries no search, so answering one would return + /// plain rows and quietly drop the search and its top K. + static Status CheckNoSearch(const std::shared_ptr& context); + + /// Loaders call this once the object exists, because `ArrowSchema()` is virtual and cannot run + /// from a constructor: it fills in this table's own read-only `SystemTableSchema`. Status AdoptOwnSchema(); friend class SystemTableLoader; diff --git a/src/paimon/core/table/system/system_table_test.cpp b/src/paimon/core/table/system/system_table_test.cpp index 0a9f85a2e..f1f0cbfb1 100644 --- a/src/paimon/core/table/system/system_table_test.cpp +++ b/src/paimon/core/table/system/system_table_test.cpp @@ -45,10 +45,13 @@ #include "paimon/fs/file_system_factory.h" #include "paimon/memory/memory_pool.h" #include "paimon/metrics.h" +#include "paimon/predicate/vector_search.h" #include "paimon/read_context.h" #include "paimon/reader/batch_reader.h" #include "paimon/result.h" +#include "paimon/scan_context.h" #include "paimon/status.h" +#include "paimon/table/source/table_read.h" #include "paimon/testing/mock/mock_file_batch_reader.h" #include "paimon/testing/utils/read_result_collector.h" #include "paimon/testing/utils/testharness.h" @@ -301,6 +304,62 @@ TEST(SystemTableTest, TestGlobalSystemTableWithoutCatalogReturnsNotImplemented) "global system table requires catalog context: tables"); } +// A system table reached through `NewXxx` directly - one a catalog handed out - is checked the +// way the static factories check the contexts they take: a null context is refused rather than +// dereferenced. +TEST(SystemTableTest, TestOperatorEntriesRefuseANullContext) { + std::map options = {{Options::FILE_SYSTEM, "local"}, + {Options::FILE_FORMAT, "orc"}}; + ASSERT_OK_AND_ASSIGN(std::shared_ptr table_schema, + CreateTableSchemaForTest(options)); + + AuditLogSystemTable audit_log(/*fs=*/nullptr, "/tmp/table", table_schema, options); + BinlogSystemTable binlog(/*fs=*/nullptr, "/tmp/table", table_schema, options); + ReadOptimizedSystemTable read_optimized(/*fs=*/nullptr, "/tmp/table", table_schema, options); + + ASSERT_NOK_WITH_MSG(audit_log.NewScan(nullptr), "scan context is null pointer"); + ASSERT_NOK_WITH_MSG(binlog.NewScan(nullptr), "scan context is null pointer"); + ASSERT_NOK_WITH_MSG(read_optimized.NewScan(nullptr), "scan context is null pointer"); + ASSERT_NOK_WITH_MSG(audit_log.NewRead(nullptr), "read context is null pointer"); + ASSERT_NOK_WITH_MSG(binlog.NewRead(nullptr), "read context is null pointer"); + ASSERT_NOK_WITH_MSG(read_optimized.NewRead(nullptr), "read context is null pointer"); +} + +// The reads these views serve build a context of their own for the data table underneath, which +// carries no search: a search the caller asked for is refused rather than quietly answered with +// plain rows. +TEST(SystemTableTest, TestNewReadRefusesASearch) { + std::map options = {{Options::FILE_SYSTEM, "local"}, + {Options::FILE_FORMAT, "orc"}}; + ASSERT_OK_AND_ASSIGN(std::shared_ptr table_schema, + CreateTableSchemaForTest(options)); + + AuditLogSystemTable audit_log(/*fs=*/nullptr, "/tmp/table", table_schema, options); + BinlogSystemTable binlog(/*fs=*/nullptr, "/tmp/table", table_schema, options); + ReadOptimizedSystemTable read_optimized(/*fs=*/nullptr, "/tmp/table", table_schema, options); + + auto make_context = [] { + auto vector_search = std::make_shared( + "v", /*limit=*/3, std::vector{1.0f}, /*pre_filter=*/nullptr, + /*predicate=*/nullptr, VectorSearch::DistanceType::COSINE, + std::map{}); + ReadContextBuilder builder("/tmp/table"); + builder.SetVectorSearch(vector_search); + return builder.Finish(); + }; + + ASSERT_OK_AND_ASSIGN(std::unique_ptr audit_log_context, make_context()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr binlog_context, make_context()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_optimized_context, make_context()); + + ASSERT_NOK_WITH_MSG(audit_log.NewRead(std::move(audit_log_context)), + "VectorSearch or FullTextSearch does not support system-table reads yet"); + ASSERT_NOK_WITH_MSG(binlog.NewRead(std::move(binlog_context)), + "VectorSearch or FullTextSearch does not support system-table reads yet"); + ASSERT_NOK_WITH_MSG(read_optimized.NewRead(std::move(read_optimized_context)), + "VectorSearch or FullTextSearch does not support system-table reads yet"); +} + namespace { /// A catalog that serves the files of every table through `GetTableFileSystem`, like one diff --git a/src/paimon/core/table/table.cpp b/src/paimon/core/table/table.cpp new file mode 100644 index 000000000..3180a48fe --- /dev/null +++ b/src/paimon/core/table/table.cpp @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/catalog/table.h" + +namespace paimon { + +Table::Table(const std::shared_ptr& file_system, const std::string& root_path, + const std::shared_ptr& schema, const Identifier& identifier, + const std::string& uuid) + : file_system_(file_system), + root_path_(root_path), + schema_(schema), + identifier_(identifier), + uuid_(uuid) {} + +} // namespace paimon diff --git a/src/paimon/core/table/table_test.cpp b/src/paimon/core/table/table_test.cpp index b6cc22859..055fc5995 100644 --- a/src/paimon/core/table/table_test.cpp +++ b/src/paimon/core/table/table_test.cpp @@ -36,10 +36,13 @@ #include "paimon/core/schema/schema_manager.h" #include "paimon/core/schema/table_schema.h" #include "paimon/core/table/file_store_table.h" +#include "paimon/core/table/source/data_split_impl.h" +#include "paimon/core/table/source/fallback_data_split.h" #include "paimon/defs.h" #include "paimon/file_store_commit.h" #include "paimon/file_store_write.h" #include "paimon/read_context.h" +#include "paimon/reader/batch_reader.h" #include "paimon/record_batch.h" #include "paimon/scan_context.h" #include "paimon/schema/schema.h" @@ -49,6 +52,8 @@ #include "paimon/table/source/table_scan.h" #include "paimon/testing/mock/mock_catalog.h" #include "paimon/testing/mock/mock_file_system.h" +#include "paimon/testing/utils/counting_cache_test_utils.h" +#include "paimon/testing/utils/read_result_collector.h" #include "paimon/testing/utils/testharness.h" #include "paimon/write_context.h" @@ -73,29 +78,6 @@ Result> CreateManagedTable( /*uuid=*/"", branch); } -/// Counts the lookups a scan makes, which is enough to tell that the cache a context carries -/// reached the scan's manifest reading rather than being dropped on the way. -class CountingCache : public Cache { - public: - Result> Get( - const std::shared_ptr& key, - std::function>(const std::shared_ptr&)> - supplier) override { - gets++; - return supplier(key); - } - Status Put(const std::shared_ptr&, const std::shared_ptr&) override { - return Status::OK(); - } - void Invalidate(const std::shared_ptr&) override {} - void InvalidateAll() override {} - size_t Size() const override { - return 0; - } - - int32_t gets = 0; -}; - /// Writes two rows through the table's own write and commit, so a scan below has data to find. void WriteAndCommitTwoRows(const std::shared_ptr
& table, const std::string& root_path) { WriteContextBuilder write_builder(root_path, "commit-user"); @@ -124,6 +106,42 @@ void WriteAndCommitTwoRows(const std::shared_ptr
& table, const std::strin ASSERT_OK(write->Close()); } +/// Writes one `id`/`v` row through the table's own write and commit, so a scan below has data +/// to find. +void WriteAndCommitRow(const std::shared_ptr
& table, const std::string& root_path, + int32_t id, int32_t v) { + WriteContextBuilder write_builder(root_path, "commit-user"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr write, + table->NewWrite(std::move(write_context))); + arrow::Int32Builder ids_builder; + arrow::Int32Builder vs_builder; + ASSERT_TRUE(ids_builder.AppendValues({id}).ok()); + ASSERT_TRUE(vs_builder.AppendValues({v}).ok()); + std::shared_ptr ids_array; + std::shared_ptr vs_array; + ASSERT_TRUE(ids_builder.Finish(&ids_array).ok()); + ASSERT_TRUE(vs_builder.Finish(&vs_array).ok()); + auto struct_array = + arrow::StructArray::Make({ids_array, vs_array}, + {arrow::field("id", arrow::int32(), /*nullable=*/false), + arrow::field("v", arrow::int32())}) + .ValueOrDie(); + ::ArrowArray arrow_array; + ASSERT_TRUE(arrow::ExportArray(*struct_array, &arrow_array).ok()); + RecordBatchBuilder batch_builder(&arrow_array); + ASSERT_OK_AND_ASSIGN(std::unique_ptr batch, batch_builder.SetBucket(0).Finish()); + ASSERT_OK(write->Write(std::move(batch))); + ASSERT_OK_AND_ASSIGN(std::vector> messages, + write->PrepareCommit()); + CommitContextBuilder commit_builder(root_path, "commit-user"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, + table->NewCommit(std::move(commit_context))); + ASSERT_OK(commit->Commit(messages)); + ASSERT_OK(write->Close()); +} + } // namespace TEST(TableTest, TestTableCarriesManagersOfItsBranch) { @@ -198,6 +216,219 @@ TEST(TableTest, TestCopyKeepsTheBranchTheTableServes) { } } +TEST(TableTest, TestACopyHandsItsDynamicOptionsToTheFallbackReader) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + const std::shared_ptr fs = dir->GetFileSystem(); + const std::string path = dir->Str() + "/tbl"; + const Identifier identifier("db", "tbl"); + + // A primary-key table whose reads fall back to the `fb` branch, and that branch's schema. + SchemaManager schema_manager(fs, path, "main"); + auto schema = arrow::schema({arrow::field("id", arrow::int32(), /*nullable=*/false)}); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr created, + schema_manager.CreateTable(schema, /*partition_keys=*/{}, /*primary_keys=*/{"id"}, + {{Options::BUCKET, "1"}, + {Options::BUCKET_KEY, "id"}, + {Options::SCAN_FALLBACK_BRANCH, "fb"}})); + SchemaManager fallback_schema_manager(fs, path, "fb"); + ASSERT_OK_AND_ASSIGN( + [[maybe_unused]] std::unique_ptr fallback_schema, + fallback_schema_manager.CreateTable(schema, /*partition_keys=*/{}, + /*primary_keys=*/{"id"}, + {{Options::BUCKET, "1"}, {Options::BUCKET_KEY, "id"}})); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr
table, + FileStoreTable::Create( + fs, path, + checked_pointer_cast(std::shared_ptr(std::move(created))), + identifier, /*uuid=*/"", /*branch=*/"main")); + + // The copy turns the sequence-number column on, and a read projects it through a context + // that carries no options of its own. The main reader serves the copy's merged options, so + // it knows the column; the fallback serves the `fb` branch's schema, whose options do not + // name it, so the copy's own dynamic options are what let the fallback reader know it too - + // without them the read as a whole fails to build. + ASSERT_OK_AND_ASSIGN(std::shared_ptr
copy, + table->Copy({{Options::KEY_VALUE_SEQUENCE_NUMBER_ENABLED, "true"}})); + ReadContextBuilder read_builder(path); + read_builder.SetReadFieldNames({"id", "_SEQUENCE_NUMBER"}); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); + ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr read, + copy->NewRead(std::move(read_context))); +} + +TEST(TableTest, TestACopyOfACopyServesTheLatestValueOfARepeatedDynamicOption) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + const std::shared_ptr fs = dir->GetFileSystem(); + const std::string path = dir->Str() + "/tbl"; + const Identifier identifier("db", "tbl"); + + // A primary-key table whose reads fall back to the `fb` branch, and that branch's schema. + SchemaManager schema_manager(fs, path, "main"); + auto schema = arrow::schema({arrow::field("id", arrow::int32(), /*nullable=*/false)}); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr created, + schema_manager.CreateTable(schema, /*partition_keys=*/{}, /*primary_keys=*/{"id"}, + {{Options::BUCKET, "1"}, + {Options::BUCKET_KEY, "id"}, + {Options::SCAN_FALLBACK_BRANCH, "fb"}})); + SchemaManager fallback_schema_manager(fs, path, "fb"); + ASSERT_OK_AND_ASSIGN( + [[maybe_unused]] std::unique_ptr fallback_schema, + fallback_schema_manager.CreateTable(schema, /*partition_keys=*/{}, + /*primary_keys=*/{"id"}, + {{Options::BUCKET, "1"}, {Options::BUCKET_KEY, "id"}})); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr
table, + FileStoreTable::Create( + fs, path, + checked_pointer_cast(std::shared_ptr(std::move(created))), + identifier, /*uuid=*/"", /*branch=*/"main")); + + // The option is turned off and then on again by the copy the read serves, so the copy's + // schema carries the latest value. The fallback branch's schema does not carry the option + // at all, so what lets its reader know the column is the copy's accumulated dynamic + // options - which must speak the same latest value the schema does, not the older one a + // first copy left, or the fallback reader fails to build while the main one succeeds. + ASSERT_OK_AND_ASSIGN(std::shared_ptr
off, + table->Copy({{Options::KEY_VALUE_SEQUENCE_NUMBER_ENABLED, "false"}})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr
copy, + off->Copy({{Options::KEY_VALUE_SEQUENCE_NUMBER_ENABLED, "true"}})); + ReadContextBuilder read_builder(path); + read_builder.SetReadFieldNames({"id", "_SEQUENCE_NUMBER"}); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); + ASSERT_OK_AND_ASSIGN([[maybe_unused]] std::unique_ptr read, + copy->NewRead(std::move(read_context))); +} + +TEST(TableTest, TestAFallbackReaderServesItsBranchsOwnSchema) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + const std::shared_ptr fs = dir->GetFileSystem(); + const std::string path = dir->Str() + "/tbl"; + const Identifier identifier("db", "tbl"); + + // The main branch is an append-only table whose reads fall back to the `fb` branch, and + // that branch is laid out as a primary-key table: a second write of the key the first + // wrote is a merge, not a duplicate. + SchemaManager main_schema_manager(fs, path, "main"); + auto row_schema = arrow::schema({arrow::field("id", arrow::int32(), /*nullable=*/false), + arrow::field("v", arrow::int32())}); + ASSERT_OK_AND_ASSIGN(std::unique_ptr main_created, + main_schema_manager.CreateTable(row_schema, /*partition_keys=*/{}, + /*primary_keys=*/{}, + {{Options::BUCKET, "1"}, + {Options::BUCKET_KEY, "id"}, + {Options::SCAN_FALLBACK_BRANCH, "fb"}})); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr
main, + FileStoreTable::Create( + fs, path, + checked_pointer_cast(std::shared_ptr(std::move(main_created))), + identifier, /*uuid=*/"", /*branch=*/"main")); + + SchemaManager fallback_schema_manager(fs, path, "fb"); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr fallback_created, + fallback_schema_manager.CreateTable(row_schema, /*partition_keys=*/{}, + /*primary_keys=*/{"id"}, + {{Options::BUCKET, "1"}, {Options::BUCKET_KEY, "id"}})); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr
fallback, + FileStoreTable::Create( + fs, path, + checked_pointer_cast(std::shared_ptr(std::move(fallback_created))), + identifier, /*uuid=*/"", /*branch=*/"fb")); + WriteAndCommitRow(fallback, path, /*id=*/1, /*v=*/10); + WriteAndCommitRow(fallback, path, /*id=*/1, /*v=*/20); + + // The read of the main table carries its fallback reader, which reads the `fb` branch's + // data through the `fb` branch's own schema - so it is the primary-key read that schema's + // layout asks for, not the append-only one the main table's kind would pick. Its splits + // are the `fb` branch's own, scanned through a table that serves `fb` and read as + // fallback ones, which is how a fallback reader is asked for its rows. + ReadContextBuilder read_builder(path); + read_builder.SetReadFieldNames({"id", "v"}); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read, main->NewRead(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, + ScanContextBuilder(path).Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr fallback_scan, + fallback->NewScan(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr fallback_plan, fallback_scan->CreatePlan()); + ASSERT_FALSE(fallback_plan->Splits().empty()); + std::vector> fallback_splits; + for (const std::shared_ptr& split : fallback_plan->Splits()) { + auto data_split = std::dynamic_pointer_cast(split); + ASSERT_TRUE(data_split != nullptr); + fallback_splits.push_back( + std::make_shared(data_split, /*is_fallback=*/true)); + } + ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_reader, + read->CreateReader(fallback_splits)); + + // The one merged row the `fb` branch's primary-key layout leaves behind - at the value + // the second write carried - is what the fallback reader answers with; the append-only + // reader the main table's kind would have picked answers with both rows, or refuses the + // split outright. + ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch batch, batch_reader->NextBatch()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr array, + ReadResultCollector::GetArray(std::move(batch))); + ASSERT_TRUE(array); + ASSERT_EQ(array->length(), 1); + auto struct_array = std::dynamic_pointer_cast(array); + ASSERT_TRUE(struct_array != nullptr); + EXPECT_EQ(struct_array->num_fields(), 3); + auto value_kind_array = std::dynamic_pointer_cast(struct_array->field(0)); + auto id_array = std::dynamic_pointer_cast(struct_array->field(1)); + auto v_array = std::dynamic_pointer_cast(struct_array->field(2)); + ASSERT_TRUE(value_kind_array != nullptr); + ASSERT_TRUE(id_array != nullptr); + ASSERT_TRUE(v_array != nullptr); + EXPECT_EQ(value_kind_array->Value(0), 0); + EXPECT_EQ(id_array->Value(0), 1); + EXPECT_EQ(v_array->Value(0), 20); + ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch eof_batch, batch_reader->NextBatch()); + EXPECT_TRUE(BatchReader::IsEofBatch(eof_batch)); +} + +TEST(TableTest, TestACopyThatMovesAnS3OptionRebuildsItsManagers) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + const std::shared_ptr fs = dir->GetFileSystem(); + const std::string path = dir->Str() + "/tbl"; + const Identifier identifier("db", "tbl"); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr
table, + CreateManagedTable( + fs, path, identifier, /*branch=*/"main", + {{Options::BUCKET, "1"}, {Options::BUCKET_KEY, "id"}, {"s3.endpoint", "http://old"}})); + auto file_store_table = std::dynamic_pointer_cast(table); + ASSERT_TRUE(file_store_table != nullptr); + + // A native `s3.` option names the file system the S3 backend resolves from, the way an + // `fs.` one does: a copy that moves it re-resolves, so the managers are rebuilt rather + // than inherited - otherwise the copy would read its metadata through the old endpoint + // while its data goes through the new one. + ASSERT_OK_AND_ASSIGN(std::shared_ptr
moved, + table->Copy({{"s3.endpoint", "http://new"}})); + auto moved_file_store_table = std::dynamic_pointer_cast(moved); + ASSERT_TRUE(moved_file_store_table != nullptr); + EXPECT_NE(moved_file_store_table->GetSchemaManager(), file_store_table->GetSchemaManager()); + EXPECT_NE(moved_file_store_table->GetSnapshotManager(), file_store_table->GetSnapshotManager()); + + // A copy whose `s3.` option says what the schema already says resolves the same file + // system, so the managers are the ones this table already holds. + ASSERT_OK_AND_ASSIGN(std::shared_ptr
kept, table->Copy({{"s3.endpoint", "http://old"}})); + auto kept_file_store_table = std::dynamic_pointer_cast(kept); + ASSERT_TRUE(kept_file_store_table != nullptr); + EXPECT_EQ(kept_file_store_table->GetSchemaManager(), file_store_table->GetSchemaManager()); + EXPECT_EQ(kept_file_store_table->GetSnapshotManager(), file_store_table->GetSnapshotManager()); +} + TEST(TableTest, TestSwitchToBranch) { auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); @@ -896,15 +1127,20 @@ TEST(TableTest, TestTheCacheAScanContextCarriesReachesTheScan) { WriteAndCommitTwoRows(table, dir->Str()); // The cache is a runtime setting the scan serves from the context rather than from the - // table's own options, so a scan built with one reads its manifests through it. - auto cache = std::make_shared(); + // table's own options, so a scan built with one reads its manifests through it. A scan reads + // the manifest list of a snapshot on its executor's threads, so the counter the cache keeps + // is guarded rather than a plain field. + auto cache = std::make_shared( + std::map{{CacheKind::MANIFEST, 64 * 1024 * 1024}, + {CacheKind::SNAPSHOT_LIVE_MANIFEST, 64 * 1024 * 1024}}); ScanContextBuilder scan_builder(dir->Str()); scan_builder.WithCache(cache); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_ctx, scan_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, table->NewScan(std::move(scan_ctx))); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); ASSERT_FALSE(plan->Splits().empty()); - ASSERT_GT(cache->gets, 0) << "the cache the scan context carries never reached the scan"; + ASSERT_GT(cache->GetCount(CacheKind::MANIFEST), 0) + << "the cache the scan context carries never reached the scan"; } TEST(TableTest, TestCopyServesTheTableWithTheGivenOptionsOnTop) { @@ -1029,33 +1265,44 @@ TEST(TableTest, TestALoadNamesTheDatabaseThePathLaysOut) { auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); const std::shared_ptr fs = dir->GetFileSystem(); + auto schema = arrow::schema({arrow::field("id", arrow::int32(), /*nullable=*/false), + arrow::field("dt", arrow::utf8())}); // A context that names no catalog knows the table by its directory: the directory above // the table's names the database when it carries the `.db` suffix, the way the Java - // `identifierFromPath` reads it, so the table the load serves is named for both - which - // the load of a directory without a schema says in the error it fails with. The option - // the context carries is what drives the load to that error: it hands the table the merge - // would build, which is where the name is spoken. + // `identifierFromPath` reads it, so the table the load serves is named for both. The + // refusal a format table open speaks names the table - and the option the context + // carries is what drives the open to it, since the schema alone opens cleanly. const std::string table_dir = dir->Str() + "/my_db.db/my_table"; ASSERT_OK(fs->Mkdirs(table_dir)); + SchemaManager schema_manager(fs, table_dir); + ASSERT_OK_AND_ASSIGN( + [[maybe_unused]] std::unique_ptr format_schema, + schema_manager.CreateTable(schema, /*partition_keys=*/{"dt"}, /*primary_keys=*/{}, + {{"type", "format-table"}})); { ScanContextBuilder builder(table_dir); - builder.AddOption("read.batch-size", "2"); + builder.AddOption(Options::REALTIME_ENABLED, "true"); ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, builder.Finish()); ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(ctx)), - "not found latest schema of my_db.my_table"); + "cannot open format table my_db.my_table"); } // A table sitting directly under the root, with no `.db` directory above it, names no // database: the table is known by its directory alone, the way it was before. const std::string loose_table_dir = dir->Str() + "/loose_table"; ASSERT_OK(fs->Mkdirs(loose_table_dir)); + SchemaManager loose_schema_manager(fs, loose_table_dir); + ASSERT_OK_AND_ASSIGN( + [[maybe_unused]] std::unique_ptr loose_schema, + loose_schema_manager.CreateTable(schema, /*partition_keys=*/{"dt"}, /*primary_keys=*/{}, + {{"type", "format-table"}})); { ScanContextBuilder builder(loose_table_dir); - builder.AddOption("read.batch-size", "2"); + builder.AddOption(Options::REALTIME_ENABLED, "true"); ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, builder.Finish()); ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(ctx)), - "not found latest schema of loose_table"); + "cannot open format table loose_table"); } } diff --git a/src/paimon/rest/rest_catalog.cpp b/src/paimon/rest/rest_catalog.cpp index 07e4b000b..fec764db3 100644 --- a/src/paimon/rest/rest_catalog.cpp +++ b/src/paimon/rest/rest_catalog.cpp @@ -425,10 +425,8 @@ Result> RestCatalog::LoadSystemTable( if (branch) { dynamic_options[Options::BRANCH] = branch.value(); } - // The system table reads the data table's own files - snapshots, manifests, data - so it is - // served by the file system that table's credentials open, which with data tokens is the - // only one that can: `GetTableFileSystem` folds the system identifier to the data table it - // belongs to and hands the catalog-wide file system out when no tokens are issued. + // The system table reads the data table's own files, so it is served by that table's own + // credentials - with data tokens the only file system that can open them. PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table_file_system, GetTableFileSystem(identifier)); return SystemTableLoader::Load(system_table_name.value(), table_file_system, data_table_path, @@ -437,9 +435,8 @@ Result> RestCatalog::LoadSystemTable( Result> RestCatalog::GetTable(const Identifier& identifier) const { // The global system tables of the "sys" database expose the catalog itself rather than any - // table the server stands behind, so they are served locally - the way `LoadTableSchema` - // serves them their schema - and never reach the managed-table load below, whose schema - // cast a read-only system table schema would not survive. + // table the server stands behind, so they are served locally and never reach the managed-table + // load below, whose schema cast a read-only system table schema would not survive. if (CatalogUtils::IsSystemDatabase(identifier.GetDatabaseName())) { PAIMON_ASSIGN_OR_RAISE(bool supported, GlobalSystemTableLoader::IsSupported(identifier.GetObjectName(), @@ -449,13 +446,21 @@ Result> RestCatalog::GetTable(const Identifier& identifie } GlobalSystemTableContext context; context.catalog = this; + // A global system table builds its rows lazily through the catalog above, so the table + // handed back has to keep it alive or the pointer dangles at the first `BuildRows`. A + // catalog not owned by a `std::shared_ptr` cannot be kept, so it is refused by name. + std::shared_ptr self = weak_from_this().lock(); + if (self == nullptr) { + return Status::Invalid( + "a global system table retains its catalog, so the catalog must be owned by " + "a std::shared_ptr before it is asked for one"); + } + context.catalog_retainer = std::move(self); context.fs = fs_; context.warehouse = warehouse_; context.catalog_options = api_->GetMergedOptions(); return GlobalSystemTableLoader::Load(identifier.GetObjectName(), context); } - // A `T$system` identifier opens the read-only view it names, which reads the data table it - // belongs to and answers for itself through the same `Table` interface. PAIMON_ASSIGN_OR_RAISE(bool is_system_table, identifier.IsSystemTable()); if (is_system_table) { PAIMON_ASSIGN_OR_RAISE(std::optional branch, identifier.GetBranchName()); @@ -467,9 +472,8 @@ Result> RestCatalog::GetTable(const Identifier& identifie std::string table_path; PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, LoadTableSchema(identifier, &uuid, &table_path)); - // The schema the same response carried says what the table is: a format table is handed - // back as the `FormatTable` it is, built from the location of the same response, rather - // than refused as the managed table it is not. + // The schema the same response carried says what the table is: a format table is handed back + // as the `FormatTable` it is rather than refused as the managed table it is not. std::shared_ptr data_schema = std::dynamic_pointer_cast(schema); if (data_schema != nullptr) { PAIMON_ASSIGN_OR_RAISE(bool is_format_table, @@ -488,8 +492,7 @@ Result> RestCatalog::GetTable(const Identifier& identifie // tokens binds to this table. PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table_file_system, GetTableFileSystem(identifier)); - // The branch the identifier carries, which the schema and path above were read on, so the - // table and its managers serve it. + // The branch the schema and path above were read on, which the table's managers then serve. PAIMON_ASSIGN_OR_RAISE(std::optional branch, identifier.GetBranchName()); return FileStoreTable::Create( table_file_system, table_path, schema, identifier, uuid, diff --git a/src/paimon/rest/rest_catalog.h b/src/paimon/rest/rest_catalog.h index 9a4e6eb06..469096e6e 100644 --- a/src/paimon/rest/rest_catalog.h +++ b/src/paimon/rest/rest_catalog.h @@ -42,7 +42,9 @@ class TableSchema; /// A catalog backed by a REST catalog server. Metadata operations are delegated to /// `RestApi`; table data is accessed through the file system configured by the /// server-merged options. -class RestCatalog : public Catalog, public VersionManagedCatalog { +class RestCatalog : public Catalog, + public VersionManagedCatalog, + public std::enable_shared_from_this { public: /// Creates the catalog: fetches and merges "/v1/config" from the server configured /// by `CatalogOptions::URI`, then builds the file system from the merged options. diff --git a/src/paimon/rest/rest_catalog_test.cpp b/src/paimon/rest/rest_catalog_test.cpp index 68081328e..5d418fc63 100644 --- a/src/paimon/rest/rest_catalog_test.cpp +++ b/src/paimon/rest/rest_catalog_test.cpp @@ -51,10 +51,17 @@ #include "paimon/file_store_commit.h" #include "paimon/fs/file_system.h" #include "paimon/memory/memory_pool.h" +#include "paimon/read_context.h" +#include "paimon/reader/batch_reader.h" #include "paimon/rest/mock_rest_server.h" #include "paimon/rest/rest_api.h" +#include "paimon/scan_context.h" #include "paimon/schema/schema.h" #include "paimon/table/format/format_table.h" +#include "paimon/table/source/plan.h" +#include "paimon/table/source/table_read.h" +#include "paimon/table/source/table_scan.h" +#include "paimon/testing/utils/read_result_collector.h" #include "paimon/testing/utils/snapshot_test_helper.h" #include "paimon/testing/utils/testharness.h" @@ -1470,6 +1477,40 @@ TEST_F(RestCatalogTest, SystemDatabase) { ASSERT_TRUE(missing.IsNotExist()) << missing.ToString(); } +TEST_F(RestCatalogTest, GlobalSystemTableRetainsItsCatalog) { + // A global system table builds its rows lazily through the catalog that served it, so the + // table handed back keeps that catalog alive: reading it after the caller let its own + // reference go still answers, rather than reading through a catalog already destroyed. + ASSERT_OK_AND_ASSIGN(std::unique_ptr created, CreateRestCatalog()); + std::shared_ptr catalog = std::move(created); + ASSERT_OK(catalog->CreateDatabase("db1", {}, /*ignore_if_exists=*/false)); + ASSERT_OK(CreateSampleTable(catalog.get(), Identifier("db1", "t1"))); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, + catalog->GetTable(Identifier("sys", "tables"))); + std::weak_ptr weak = catalog; + catalog.reset(); + ASSERT_FALSE(weak.expired()) << "the global system table does not retain its catalog"; + + // The rows are built through the retained catalog, so they still answer once the caller's + // own reference is gone: the one table the catalog stands behind shows up in them. + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, + ScanContextBuilder("sys/tables").Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, table->NewScan(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); + ASSERT_EQ(plan->Splits().size(), 1); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, + ReadContextBuilder("sys/tables").Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read, table->NewRead(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_reader, + read->CreateReader(plan->Splits())); + ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch batch, batch_reader->NextBatch()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr array, + ReadResultCollector::GetArray(std::move(batch))); + ASSERT_TRUE(array); + ASSERT_GT(array->length(), 0); +} + TEST_F(RestCatalogTest, ListSnapshots) { ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, CreateRestCatalog()); ASSERT_OK(catalog->CreateDatabase("db1", {}, /*ignore_if_exists=*/false)); diff --git a/test/inte/realtime_write_inte_test.cpp b/test/inte/realtime_write_inte_test.cpp index fd9b49650..5de6c9fa7 100644 --- a/test/inte/realtime_write_inte_test.cpp +++ b/test/inte/realtime_write_inte_test.cpp @@ -43,7 +43,6 @@ #include "gtest/gtest.h" #include "paimon/catalog/catalog.h" #include "paimon/catalog/identifier.h" -#include "paimon/catalog/table.h" #include "paimon/commit_context.h" #include "paimon/common/data/variant/variant_type_utils.h" #include "paimon/common/table/special_fields.h" @@ -560,41 +559,6 @@ class RealtimeWriteInteTest : public ::testing::Test { return FileStoreWrite::Create(std::move(context)); } - // A writer that means to change an option the table was created with writes through a copy - // carrying it, which the context then agrees with. - Result> CopiedTable( - const std::map& overrides) const { - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr catalog, Catalog::Create(dir_->Str(), {})); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, - catalog->GetTable(Identifier("foo", "bar"))); - return table->Copy(overrides); - } - - // Evolves the table schema to the current options with its fields unchanged, the way an - // ALTER TABLE changing an option would: the table itself then serves the changed option. - // The fields are carried over from the latest schema so their attributes survive: rebuilt - // raw fields would lose the non-nullable primary keys the table schema enforces. - Status EvolveSchemaWithOptions() const { - SchemaManager schema_manager(dir_->GetFileSystem(), table_path_); - PAIMON_ASSIGN_OR_RAISE(std::optional> latest, - schema_manager.Latest()); - if (!latest.has_value()) { - return Status::Invalid("table schema does not exist"); - } - return TestHelper::WriteNextSchema(dir_->GetFileSystem(), table_path_, - latest.value()->Fields(), - latest.value()->HighestFieldId(), options_); - } - - Result> CreateRealtimeWriter( - const std::shared_ptr& realtime_context, - const std::shared_ptr
& table) const { - WriteContextBuilder builder(table_path_, commit_user_); - builder.SetOptions(options_).WithStreamingMode(true).WithRealtimeContext(realtime_context); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, builder.Finish()); - return table->NewWrite(std::move(context)); - } - Result> CreateRealtimeWriter() const { PAIMON_ASSIGN_OR_RAISE(std::shared_ptr realtime_context, RealtimeContext::Create()); @@ -1285,36 +1249,32 @@ TEST_F(RealtimeWriteInteTest, TestRealtimeOperationsRequireEnabledOption) { disabled_options[Options::REALTIME_ENABLED] = "false"; ASSERT_OK_AND_ASSIGN(std::shared_ptr realtime_context, RealtimeContext::Create()); - // The table was created enabled, so each operation below goes through a copy carrying it - // disabled, whose contexts then agree: each reports that real-time needs the option itself. - ASSERT_OK_AND_ASSIGN(std::shared_ptr
disabled_table, - CopiedTable({{Options::REALTIME_ENABLED, "false"}})); WriteContextBuilder write_builder(table_path_, commit_user_); write_builder.SetOptions(disabled_options) .WithStreamingMode(true) .WithRealtimeContext(realtime_context); ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); - ASSERT_NOK_WITH_MSG(disabled_table->NewWrite(std::move(write_context)), + ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(std::move(write_context)), "real-time write requires realtime.enabled=true"); ScanContextBuilder scan_builder(table_path_); scan_builder.SetOptions(disabled_options).WithRealtimeContext(realtime_context); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_builder.Finish()); - ASSERT_NOK_WITH_MSG(disabled_table->NewScan(std::move(scan_context)), + ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(scan_context)), "real-time scan requires realtime.enabled=true"); ReadContextBuilder read_builder(table_path_); read_builder.SetOptions(disabled_options).WithRealtimeContext(realtime_context); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_builder.Finish()); - ASSERT_NOK_WITH_MSG(disabled_table->NewRead(std::move(read_context)), + ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(read_context)), "real-time read requires realtime.enabled=true"); CommitContextBuilder commit_builder(table_path_, commit_user_); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.SetOptions(disabled_options).Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, - disabled_table->NewCommit(std::move(commit_context))); + FileStoreCommit::Create(std::move(commit_context))); ASSERT_NOK_WITH_MSG(commit->CommitWithProgress(/*realtime_commits=*/{}, /*commit_identifier=*/0, /*watermark=*/std::nullopt), @@ -2124,9 +2084,6 @@ TEST_F(RealtimeWriteInteTest, TestRealtimeDeduplicateAcrossMultipleFileIndexStor ASSERT_OK(embedded_writer->Close()); options_[Options::FILE_INDEX_IN_MANIFEST_THRESHOLD] = "1B"; - // The tighter threshold is a change the table itself carries, the way an ALTER TABLE would: - // the schema is evolved to it, so every later write, commit, and read serves it. - ASSERT_OK(EvolveSchemaWithOptions()); ASSERT_OK_AND_ASSIGN(std::unique_ptr external_writer, CreateRealtimeWriter(realtime_context)); ASSERT_OK_AND_ASSIGN(std::unique_ptr external_batch, @@ -3809,9 +3766,6 @@ TEST_F(RealtimeWriteInteTest, TestAppendRealtimeReadUsesEmbeddedAndExternalBitma ASSERT_OK(embedded_writer->Close()); options_[Options::FILE_INDEX_IN_MANIFEST_THRESHOLD] = "1B"; - // The tighter threshold is a change the table itself carries, the way an ALTER TABLE would: - // the schema is evolved to it, so every later write, commit, and read serves it. - ASSERT_OK(EvolveSchemaWithOptions()); ASSERT_OK_AND_ASSIGN(std::unique_ptr external_writer, CreateRealtimeWriter(realtime_context)); ASSERT_OK_AND_ASSIGN( @@ -4129,9 +4083,6 @@ TEST_F(RealtimeWriteInteTest, TestPkDvRealtimeReadUsesEmbeddedAndExternalBitmapI ASSERT_OK(embedded_writer->Close()); options_[Options::FILE_INDEX_IN_MANIFEST_THRESHOLD] = "1B"; - // The tighter threshold is a change the table itself carries, the way an ALTER TABLE would: - // the schema is evolved to it, so every later write, commit, and read serves it. - ASSERT_OK(EvolveSchemaWithOptions()); ASSERT_OK_AND_ASSIGN(std::unique_ptr external_writer, CreateRealtimeWriter(realtime_context)); ASSERT_OK_AND_ASSIGN(std::unique_ptr external_batch, diff --git a/test/inte/scan_and_read_inte_test.cpp b/test/inte/scan_and_read_inte_test.cpp index 4fa4e1da1..b7cc73b96 100644 --- a/test/inte/scan_and_read_inte_test.cpp +++ b/test/inte/scan_and_read_inte_test.cpp @@ -36,9 +36,6 @@ #include "fmt/ranges.h" #include "gtest/gtest.h" #include "paimon/bucket/bucket_id_calculator.h" -#include "paimon/catalog/catalog.h" -#include "paimon/catalog/identifier.h" -#include "paimon/catalog/table.h" #include "paimon/common/factories/io_hook.h" #include "paimon/common/io/cache/lru_cache.h" #include "paimon/common/table/special_fields.h" @@ -799,36 +796,19 @@ TEST_P(ScanAndReadInteTest, TestWithAppendBucketKeyPointLookup) { } ASSERT_GE(other_index, 0); // These keys share a current two-bucket selection but need different historical buckets. - // The data was written with four buckets, so reading a rescaled layout reads through a - // copy of the table carrying the rescaled bucket count: an option on the context would - // contradict the table the catalog loaded. - ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, - Catalog::Create(test_dir->Str(), {})); - ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, - catalog->GetTable(Identifier("foo", "bar"))); for (int32_t lookup_index : {32, other_index}) { + // The data was written with four buckets. Read it with rescaled table options too. for (const std::string current_bucket_count : {"2", "4", "8", "17"}) { SCOPED_TRACE(current_bucket_count); const std::string lookup_key = fmt::format("key{:03}", lookup_index); auto lookup_predicate = PredicateBuilder::Equal( 0, "rowkey", field_type, Literal(field_type, lookup_key.data(), lookup_key.size())); - // The rescaled bucket count and, with it, the cache settings `FinishScanContext` - // also puts on the context: a table already held takes its runtime settings - // through the copy it is made of, and the context carries them only to agree - // with it. - std::map dynamic_options{ - {Options::BUCKET, current_bucket_count}}; - if (EnableSnapshotLiveManifestCache()) { - dynamic_options.emplace(Options::SCAN_MANIFEST_ENTRY_CACHE_MAX_SNAPSHOTS, "3"); - } - ASSERT_OK_AND_ASSIGN(std::shared_ptr
rescaled_table, - table->Copy(dynamic_options)); ScanContextBuilder scan_builder(table_path); scan_builder.SetPredicate(lookup_predicate) .AddOption(Options::BUCKET, current_bucket_count); ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_builder)); - ASSERT_OK_AND_ASSIGN(auto scan, rescaled_table->NewScan(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto scan, TableScan::Create(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(auto plan, scan->CreatePlan()); if (EnableSnapshotLiveManifestCache()) { ASSERT_OK_AND_ASSIGN( @@ -840,7 +820,7 @@ TEST_P(ScanAndReadInteTest, TestWithAppendBucketKeyPointLookup) { .AddOption(Options::BUCKET, current_bucket_count); ASSERT_OK_AND_ASSIGN(auto cached_context, FinishScanContext(cached_builder)); ASSERT_OK_AND_ASSIGN(auto cached_scan, - rescaled_table->NewScan(std::move(cached_context))); + TableScan::Create(std::move(cached_context))); ASSERT_OK_AND_ASSIGN(plan, cached_scan->CreatePlan()); ASSERT_OK_AND_ASSIGN(uint64_t hit, cached_scan->GetMetrics()->GetCounter( ScanMetrics::LAST_SNAPSHOT_CACHE_HIT)); @@ -3106,21 +3086,12 @@ TEST_P(ScanAndReadInteTest, TestScanAndReadWithDisableIndex) { PredicateBuilder::Equal(/*field_index=*/0, /*field_name=*/"f0", FieldType::STRING, Literal(FieldType::STRING, "Alice0", 6)); - // The table was loaded with its index enabled, so the index-disabled pass below goes through - // a copy carrying it disabled: an option on the context would contradict the table. - ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, - Catalog::Create(paimon::test::GetDataDir() + file_format, {})); - ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, - catalog->GetTable(Identifier("append_with_bitmap", "append_with_bitmap"))); - ASSERT_OK_AND_ASSIGN(std::shared_ptr
disabled_table, - table->Copy({{"file-index.read.enabled", "false"}})); - // scan ScanContextBuilder scan_context_builder(table_path); ASSERT_OK_AND_ASSIGN( auto scan_context, scan_context_builder.AddOption("file-index.read.enabled", "false").Finish()); - ASSERT_OK_AND_ASSIGN(auto table_scan, disabled_table->NewScan(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 1); @@ -3131,7 +3102,7 @@ TEST_P(ScanAndReadInteTest, TestScanAndReadWithDisableIndex) { read_context_builder.AddOption("file-index.read.enabled", "false"); AddReadOptionsForPrefetch(&read_context_builder); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, disabled_table->NewRead(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(splits)); ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(std::move(batch_reader))); @@ -3187,13 +3158,6 @@ TEST(ScanAndReadFileIndexInteTest, TestPkDvEmbeddedBitmapFiltersScanAndRead) { helper->WriteAndCommit(std::move(batch), /*commit_identifier=*/0, std::nullopt)); ASSERT_FALSE(commit_messages.empty()); - // The statistics-only baseline below disables the file index the table carries, so it scans - // through a copy with it disabled: an option on the context would contradict the table. - ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, Catalog::Create(dir->Str(), {})); - ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, catalog->GetTable(Identifier("foo", "bar"))); - ASSERT_OK_AND_ASSIGN(std::shared_ptr
stats_only_table, - table->Copy({{Options::FILE_INDEX_READ_ENABLED, "false"}})); - // 15 falls inside the file's [10, 30] min/max range, so statistics alone must retain the // file. Disabling file-index reads verifies that baseline explicitly. const std::shared_ptr absent_predicate = PredicateBuilder::Equal( @@ -3205,7 +3169,7 @@ TEST(ScanAndReadFileIndexInteTest, TestPkDvEmbeddedBitmapFiltersScanAndRead) { ASSERT_OK_AND_ASSIGN(std::unique_ptr stats_only_scan_context, stats_only_scan_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr stats_only_scan, - stats_only_table->NewScan(std::move(stats_only_scan_context))); + TableScan::Create(std::move(stats_only_scan_context))); ASSERT_OK_AND_ASSIGN(std::shared_ptr stats_only_plan, stats_only_scan->CreatePlan()); ASSERT_FALSE(stats_only_plan->Splits().empty()); @@ -3298,13 +3262,6 @@ TEST(ScanAndReadFileIndexInteTest, TestPkDvReconstructsFileIndexPredicateAfterSc const std::shared_ptr predicate = PredicateBuilder::Equal( /*field_index=*/1, /*field_name=*/"renamed_value", FieldType::INT, Literal(15)); - // The statistics-only baseline below disables the file index the table carries, so it scans - // through a copy with it disabled: an option on the context would contradict the table. - ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, Catalog::Create(dir->Str(), {})); - ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, catalog->GetTable(Identifier("foo", "bar"))); - ASSERT_OK_AND_ASSIGN(std::shared_ptr
stats_only_table, - table->Copy({{Options::FILE_INDEX_READ_ENABLED, "false"}})); - // The evolved min/max range is still [10, 30], so statistics alone retain the old file. ScanContextBuilder stats_only_scan_builder(table_path); stats_only_scan_builder.SetOptions(options) @@ -3313,7 +3270,7 @@ TEST(ScanAndReadFileIndexInteTest, TestPkDvReconstructsFileIndexPredicateAfterSc ASSERT_OK_AND_ASSIGN(std::unique_ptr stats_only_scan_context, stats_only_scan_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr stats_only_scan, - stats_only_table->NewScan(std::move(stats_only_scan_context))); + TableScan::Create(std::move(stats_only_scan_context))); ASSERT_OK_AND_ASSIGN(std::shared_ptr stats_only_plan, stats_only_scan->CreatePlan()); ASSERT_FALSE(stats_only_plan->Splits().empty()); @@ -4064,31 +4021,16 @@ TEST_P(ScanAndReadInteTest, TestWithPKBucketSelectByPredicate) { auto predicate = PredicateBuilder::Equal(/*field_index=*/2, /*field_name=*/"f2", FieldType::INT, Literal(static_cast(0))); - // Historical files use two buckets even when the current option has changed, so the scan - // goes through a copy of the table carrying the bucket count it selects by: an option on - // the context would contradict the table the catalog loaded. - ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, - Catalog::Create(paimon::test::GetDataDir() + file_format, {})); - ASSERT_OK_AND_ASSIGN( - std::shared_ptr
table, - catalog->GetTable(Identifier("pk_table_scan_and_read_dv", "pk_table_scan_and_read_dv"))); + // Historical files use two buckets even when the current option has changed. for (const std::string current_bucket_count : {"2", "4", "8", "17"}) { SCOPED_TRACE(current_bucket_count); - // The rescaled bucket count and, with it, the cache settings `FinishScanContext` also - // puts on the context: a table already held takes its runtime settings through the copy - // it is made of, and the context carries them only to agree with it. - std::map dynamic_options{{Options::BUCKET, current_bucket_count}}; - if (EnableSnapshotLiveManifestCache()) { - dynamic_options.emplace(Options::SCAN_MANIFEST_ENTRY_CACHE_MAX_SNAPSHOTS, "3"); - } - ASSERT_OK_AND_ASSIGN(std::shared_ptr
rescaled_table, table->Copy(dynamic_options)); ScanContextBuilder scan_context_builder(table_path); scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "6"); scan_context_builder.AddOption(Options::BUCKET, current_bucket_count); scan_context_builder.SetPartitionFilter({{{"f1", "10"}}}); scan_context_builder.SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); - ASSERT_OK_AND_ASSIGN(auto table_scan, rescaled_table->NewScan(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ReadContextBuilder read_context_builder(table_path); AddReadOptionsForPrefetch(&read_context_builder); @@ -4107,8 +4049,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKBucketSelectByPredicate) { .SetPartitionFilter({{{"f1", "10"}}}) .SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(auto cached_context, FinishScanContext(cached_builder)); - ASSERT_OK_AND_ASSIGN(auto cached_scan, - rescaled_table->NewScan(std::move(cached_context))); + ASSERT_OK_AND_ASSIGN(auto cached_scan, TableScan::Create(std::move(cached_context))); ASSERT_OK_AND_ASSIGN(result_plan, cached_scan->CreatePlan()); ASSERT_OK_AND_ASSIGN(uint64_t hit, cached_scan->GetMetrics()->GetCounter( ScanMetrics::LAST_SNAPSHOT_CACHE_HIT)); diff --git a/test/inte/scan_inte_test.cpp b/test/inte/scan_inte_test.cpp index 1502c39a7..3f949484d 100644 --- a/test/inte/scan_inte_test.cpp +++ b/test/inte/scan_inte_test.cpp @@ -32,9 +32,6 @@ #include "arrow/api.h" #include "gtest/gtest.h" -#include "paimon/catalog/catalog.h" -#include "paimon/catalog/identifier.h" -#include "paimon/catalog/table.h" #include "paimon/common/data/binary_row.h" #include "paimon/common/data/data_define.h" #include "paimon/common/utils/decimal_utils.h" @@ -1164,18 +1161,10 @@ TEST_P(ScanInteTest, TestScanAppendWithInvalidOptions) { "scan.snapshot-id or scan.tag-name must be set when startup mode is FROM_SNAPSHOT"); } { - // `bucket` = -2 is refused by the scan itself, which a copy of the table carrying it - // reaches: an option on the context would contradict the table the catalog loaded, - // which a scan through that table refuses. - ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, - Catalog::Create(paimon::test::GetDataDir() + "orc", {})); - ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, - catalog->GetTable(Identifier("append_09", "append_09"))); - ASSERT_OK_AND_ASSIGN(std::shared_ptr
copied, table->Copy({{Options::BUCKET, "-2"}})); ScanContextBuilder context_builder(table_path); - context_builder.WithStreamingMode(true); + context_builder.AddOption(Options::BUCKET, "-2").WithStreamingMode(true); ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_NOK_WITH_MSG(copied->NewScan(std::move(scan_context)), + ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(scan_context)), "do not support bucket=-2 in scan process"); } } @@ -1737,18 +1726,10 @@ TEST_P(ScanInteTest, TestScanPKWithSnapshot1WithBucketStats) { TEST_P(ScanInteTest, TestScanPKWithInvalidOptions) { std::string table_path = paimon::test::GetDataDir() + "orc/pk_table_with_total_buckets.db/pk_table_with_total_buckets"; - // Like the append case above, the invalid bucket count rides on a copy of the table: the - // scan refuses it where the bucket count is validated. - ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, - Catalog::Create(paimon::test::GetDataDir() + "orc", {})); - ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, - catalog->GetTable(Identifier("pk_table_with_total_buckets", - "pk_table_with_total_buckets"))); - ASSERT_OK_AND_ASSIGN(std::shared_ptr
copied, table->Copy({{Options::BUCKET, "-1"}})); ScanContextBuilder context_builder(table_path); - context_builder.WithStreamingMode(true); + context_builder.AddOption(Options::BUCKET, "-1").WithStreamingMode(true); ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); - ASSERT_NOK_WITH_MSG(copied->NewScan(std::move(scan_context)), + ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(scan_context)), "do not support pk table bucket=-1 in scan process"); } diff --git a/test/inte/write_and_read_inte_with_index_test.cpp b/test/inte/write_and_read_inte_with_index_test.cpp index 9123a3385..efda3de03 100644 --- a/test/inte/write_and_read_inte_with_index_test.cpp +++ b/test/inte/write_and_read_inte_with_index_test.cpp @@ -1250,7 +1250,8 @@ TEST_P(WriteAndReadInteWithIndexTest, TestReadWithLimits) { PredicateBuilder::NotEqual(/*field_index=*/0, /*field_name=*/"f0", FieldType::STRING, Literal(FieldType::STRING, "Alice", 5)); ReadContextBuilder context_builder(path); - context_builder.AddOption("test.enable-adaptive-prefetch-strategy", "false") + context_builder.AddOption(Options::FILE_FORMAT, "orc") + .AddOption("test.enable-adaptive-prefetch-strategy", "false") .AddOption(Options::READ_BATCH_SIZE, "1") .AddOption("orc.read.enable-metrics", "true") .SetPredicate(predicate); From 13fba3780476ce4177d2197e0d7f62816bdd3047 Mon Sep 17 00:00:00 2001 From: "yonghao.fyh" Date: Tue, 27 Oct 2026 09:42:57 +0800 Subject: [PATCH 15/17] fix --- src/paimon/core/catalog/snapshot_commit.cpp | 59 +++ .../core/table/file_store_table_factory.cpp | 346 ++++++++++++++++++ .../core/table/file_store_table_factory.h | 51 +++ src/paimon/core/utils/context_utils.h | 51 +++ 4 files changed, 507 insertions(+) create mode 100644 src/paimon/core/catalog/snapshot_commit.cpp create mode 100644 src/paimon/core/table/file_store_table_factory.cpp create mode 100644 src/paimon/core/table/file_store_table_factory.h create mode 100644 src/paimon/core/utils/context_utils.h diff --git a/src/paimon/core/catalog/snapshot_commit.cpp b/src/paimon/core/catalog/snapshot_commit.cpp new file mode 100644 index 000000000..35efe767c --- /dev/null +++ b/src/paimon/core/catalog/snapshot_commit.cpp @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/core/catalog/snapshot_commit.h" + +#include +#include + +#include "paimon/core/catalog/catalog_snapshot_commit.h" +#include "paimon/core/catalog/renaming_snapshot_commit.h" +#include "paimon/core/catalog/version_managed_catalog.h" +#include "paimon/result.h" +#include "paimon/status.h" + +namespace paimon { + +Result> SnapshotCommit::Create( + const std::shared_ptr& catalog, const std::optional& catalog_identifier, + const std::optional& table_id, bool use_rest_catalog_commit, + const std::shared_ptr& file_system, + const std::shared_ptr& snapshot_manager) { + if (AsVersionManaged(catalog) != nullptr) { + if (!catalog_identifier) { + return Status::Invalid("a catalog commit requires a table identifier"); + } + // Reuse the caller's UUID so table recreation cannot redirect this commit. + return std::shared_ptr( + std::make_shared(catalog, catalog_identifier.value(), table_id)); + } + if (use_rest_catalog_commit) { + return std::shared_ptr(std::make_shared(table_id)); + } + // Catalogs without version management may still supply a table ID. + if (table_id && catalog == nullptr) { + return Status::Invalid( + "a commit which publishes the snapshot by writing it to the table directory sends no " + "commit table request, so there is nowhere to put a table id; commit through a " + "catalog with WithCatalog() or ask for the request with UseRESTCatalogCommit()"); + } + return std::shared_ptr( + std::make_shared(file_system, snapshot_manager)); +} + +} // namespace paimon diff --git a/src/paimon/core/table/file_store_table_factory.cpp b/src/paimon/core/table/file_store_table_factory.cpp new file mode 100644 index 000000000..2b5c9a40c --- /dev/null +++ b/src/paimon/core/table/file_store_table_factory.cpp @@ -0,0 +1,346 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/core/table/file_store_table_factory.h" + +#include +#include +#include +#include +#include + +#include "fmt/format.h" +#include "paimon/catalog/catalog.h" +#include "paimon/catalog/identifier.h" +#include "paimon/catalog/table.h" +#include "paimon/commit_context.h" +#include "paimon/common/utils/checked_cast.h" +#include "paimon/core/catalog/catalog_utils.h" +#include "paimon/core/catalog/version_managed_catalog.h" +#include "paimon/core/core_options.h" +#include "paimon/core/options/table_type.h" +#include "paimon/core/schema/schema_manager.h" +#include "paimon/core/schema/table_schema.h" +#include "paimon/core/table/file_store.h" +#include "paimon/core/table/file_store_table.h" +#include "paimon/core/table/system/system_table.h" +#include "paimon/core/utils/branch_manager.h" +#include "paimon/fs/file_system.h" +#include "paimon/predicate/predicate_utils.h" +#include "paimon/read_context.h" +#include "paimon/result.h" +#include "paimon/scan_context.h" +#include "paimon/schema/schema.h" +#include "paimon/status.h" +#include "paimon/table/format/format_table.h" +#include "paimon/write_context.h" + +namespace paimon { + +namespace { + +/// Recognises a format table behind `table_path`, or returns null when it is not one. The scan, +/// read, write and commit entry points below all dispatch on the same question, so one place +/// answers it and they cannot disagree. Null is also the answer when no schema was found under +/// the path at all: what to say about a table that is not there is the managed path's to decide. +/// +/// The table type is read from the schema, never from `options`: `type` is structural, and one +/// call's options must not decide what kind of table this is. +Result> TryLoadFormatTable( + const std::shared_ptr& file_system, const std::string& table_path, + const std::string& branch, const std::map& options, + const std::shared_ptr& table_schema) { + if (table_schema == nullptr) { + return std::shared_ptr(); + } + // From the schema alone: see `FormatTable::Create()`. + PAIMON_ASSIGN_OR_RAISE(bool is_format_table, + TableTypeDefine::IsFormatTable(table_schema->Options())); + if (!is_format_table) { + return std::shared_ptr(); + } + // A format table has no branches, so it is refused rather than served from the table path + // under another branch's name. See `SchemaValidation::ValidateFormatTableSchema()`. + if (BranchManager::NormalizeBranch(branch) != BranchManager::DEFAULT_MAIN_BRANCH) { + return Status::Invalid( + fmt::format("a format table has no branches, so {} cannot be read on branch '{}'", + table_path, branch)); + } + Identifier identifier = CatalogUtils::IdentifierFromPath(table_path, branch); + return FormatTable::Create(file_system, table_path, identifier, + checked_pointer_cast(table_schema), + /*location_carries_paimon_metadata=*/true, options); +} + +/// The options a context carries, merged into the table the load hands back: a table is the +/// product of the load, so the options speak there - the way `Copy` merges them - and `NewXxx` on +/// the product serves them without merging again. A caller holding a table already changes it +/// through `Copy` itself, and an option that contradicts the table that way is refused. +Result> WithContextOptions( + const std::shared_ptr
& table, const std::shared_ptr& schema, + const std::map& options) { + if (table == nullptr || schema == nullptr || options.empty()) { + return table; + } + const std::map& schema_options = schema->Options(); + for (const auto& [key, value] : options) { + auto schema_option = schema_options.find(key); + // A key the schema does not carry, or carries differently: the merge has something to do. + if (schema_option == schema_options.end() || schema_option->second != value) { + return table->Copy(options); + } + } + return table; +} + +/// The tail every `Create` ends with: the schema one of them read says what the path names. A +/// format table is handed back as the `FormatTable` it is, everything else as the `FileStoreTable` +/// a managed table is, carrying the schema for its `NewXxx` to build from. A null schema is handed +/// over as it is - see `TryLoadFormatTable`. +/// +/// @param file_system The file system the schema above was read through, which a format table +/// is served by - it has no schema options of its own to name another one. +/// @param explicit_file_system The file system the context supplied, which a managed table is +/// opened through when there is one. A null one leaves the table to resolve from its +/// schema's own access configuration rather than being pinned to the file system this load +/// happened to read the schema through. +Result> TableOfLoadedSchema( + const std::shared_ptr& file_system, + const std::shared_ptr& explicit_file_system, const std::string& path, + const std::string& branch, const std::map& options, + const std::shared_ptr& schema_manager, + const std::shared_ptr& latest_schema, const std::optional& identifier, + const std::string& uuid = "", const std::map& fs_scheme_map = {}) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr format_table, + TryLoadFormatTable(file_system, path, branch, options, latest_schema)); + if (format_table != nullptr) { + return std::shared_ptr
(std::move(format_table)); + } + // The branch the schema was read on is folded into the object name. + Identifier table_identifier = + identifier.value_or(CatalogUtils::IdentifierFromPath(path, branch)); + // The manager was built over the file system the context's own options resolve, while the + // store resolves the schema's: a schema carrying access configuration names a backend the + // context's options never did, and a manager kept through that disagreement would read the + // table's metadata through the wrong one. The schema is in hand already, so dropping the + // manager costs its cache alone. A context that supplied a file system wins on both sides and + // keeps it; a scheme map only the store's resolution takes part in is the same disagreement. + std::shared_ptr manager = schema_manager; + if (manager != nullptr && latest_schema != nullptr && explicit_file_system == nullptr && + (!fs_scheme_map.empty() || + !FileStore::FileSystemResolutionAgrees(latest_schema->Options(), options))) { + manager = nullptr; + } + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table, + FileStoreTable::Create(explicit_file_system, path, latest_schema, + table_identifier, uuid, branch, manager, + /*snapshot_manager=*/nullptr, fs_scheme_map)); + return WithContextOptions(table, latest_schema, options); +} + +/// The managed table a context names behind a path: the schema is read through a manager of the +/// context's branch - handed over instead when the caller already read it - and what the schema +/// says is there decides the table `TableOfLoadedSchema` hands back. +/// +/// @param schema_json The schema of the table, when the caller already read it, or null when it +/// has to be read here. It is honoured on the main branch only; a branch publishes its own +/// under `branch/branch-`, which is where the read below goes. +/// @param explicit_file_system The file system the context supplied, which the managed table the +/// schema describes is opened through - see `TableOfLoadedSchema`. +Result> ManagedTableOf( + const std::shared_ptr& file_system, + const std::shared_ptr& explicit_file_system, const std::string& path, + const std::string& branch, const std::map& options, + const std::optional& schema_json, const std::optional& identifier, + const std::string& uuid = "", const std::map& fs_scheme_map = {}) { + auto schema_manager = std::make_shared(file_system, path, branch); + std::shared_ptr latest_schema; + if (branch == BranchManager::DEFAULT_MAIN_BRANCH && schema_json) { + // Handing the schema over only saves reading it. A context always names a table path and + // this library writes the metadata under it, so `schema` and `branch` stay metadata. + PAIMON_ASSIGN_OR_RAISE(latest_schema, TableSchema::CreateFromJson(schema_json.value())); + } else { + PAIMON_ASSIGN_OR_RAISE(std::optional> latest, + schema_manager->Latest()); + if (latest) { + latest_schema = *latest; + } + } + return TableOfLoadedSchema(file_system, explicit_file_system, path, branch, options, + schema_manager, latest_schema, identifier, uuid, fs_scheme_map); +} + +/// The schema a catalog may hold for the table, read from it on the main branch only and null +/// otherwise: a branch publishes its own under `branch/branch-`, which is where +/// `ManagedTableOf` reads instead, so a catalog answering for no branch schema does not fail the +/// `kind` operation to one. `kind` also names the operator the messages speak of. +Result> CatalogMainBranchSchemaJson( + const std::shared_ptr& catalog, const std::optional& catalog_identifier, + const std::string& branch, const char* kind) { + if (catalog == nullptr) { + return std::optional(); + } + if (!catalog_identifier) { + return Status::Invalid(fmt::format("a catalog {} requires a table identifier", kind)); + } + if (BranchManager::IsMainBranch(branch)) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, + catalog->LoadTableSchema(catalog_identifier.value())); + PAIMON_ASSIGN_OR_RAISE(std::string schema_json, schema->GetJsonSchema()); + return std::optional(std::move(schema_json)); + } + return std::optional(); +} + +} // namespace + +/// The table a scan context names: the format table the builder was handed, a system table behind +/// a `T$system` path, or the table the schema under the context's path says is there. +Result> FileStoreTableFactory::Create(const ScanContext& ctx) { + if (ctx.GetFormatTable() != nullptr) { + return std::shared_ptr
(ctx.GetFormatTable()); + } + PAIMON_ASSIGN_OR_RAISE(CoreOptions tmp_options, + CoreOptions::FromMap(ctx.GetOptions(), ctx.GetSpecificFileSystem(), {})); + PAIMON_ASSIGN_OR_RAISE(std::optional system_table_path, + SystemTableLoader::TryParsePath(ctx.GetPath())); + if (system_table_path) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr system_table, + SystemTableLoader::LoadFromPath(tmp_options.GetFileSystem(), + ctx.GetPath(), ctx.GetOptions())); + return std::shared_ptr
(std::move(system_table)); + } + // One `TableScan` interface serves a format table too, the way Java Paimon serves both + // through one `ReadBuilder`; a format table is planned by listing directories instead. + const std::string branch = BranchManager::NormalizeBranch(tmp_options.GetBranch()); + return ManagedTableOf(tmp_options.GetFileSystem(), ctx.GetSpecificFileSystem(), ctx.GetPath(), + branch, ctx.GetOptions(), ctx.GetSpecificTableSchema(), + ctx.GetIdentifier()); +} + +/// The table a read context names: the format table the builder was handed, a system table behind +/// a `T$system` path, or the table the schema under the context's path says is there. +Result> FileStoreTableFactory::Create(const ReadContext& ctx) { + if (ctx.GetFormatTable() != nullptr) { + return std::shared_ptr
(ctx.GetFormatTable()); + } + PAIMON_ASSIGN_OR_RAISE(CoreOptions tmp_core_options, + CoreOptions::FromMap(ctx.GetOptions(), ctx.GetSpecificFileSystem(), + ctx.GetFileSystemSchemeToIdentifierMap())); + PAIMON_ASSIGN_OR_RAISE(std::optional system_table_path, + SystemTableLoader::TryParsePath(ctx.GetPath())); + if (system_table_path) { + if (PredicateUtils::ContainsFullTextMatch(ctx.GetPredicate())) { + return Status::NotImplemented( + "FullTextMatch predicate execution is not implemented yet"); + } + if (ctx.HasVectorOrFullTextSearch()) { + return Status::NotImplemented( + "VectorSearch or FullTextSearch is not supported for system tables"); + } + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr system_table, + SystemTableLoader::LoadFromPath(tmp_core_options.GetFileSystem(), + ctx.GetPath(), ctx.GetOptions())); + return std::shared_ptr
(std::move(system_table)); + } + + // A format table has no manifest and a file another engine wrote carries no field ids, so how + // a row is put back together is its own; how a file is opened is not - both paths go through + // `DataFileReaderFactory`. + const std::string branch = ctx.GetBranch(); + // The scheme map the schema was read through, so the table's data resolves custom schemes the + // same way its metadata does. + return ManagedTableOf(tmp_core_options.GetFileSystem(), ctx.GetSpecificFileSystem(), + ctx.GetPath(), branch, ctx.GetOptions(), ctx.GetSpecificTableSchema(), + ctx.GetIdentifier(), /*uuid=*/"", + ctx.GetFileSystemSchemeToIdentifierMap()); +} + +/// The table a write context names: the format table the builder was handed, or the table the +/// schema under the context's root path says is there. +Result> FileStoreTableFactory::Create(const WriteContext& ctx) { + if (ctx.GetFormatTable() != nullptr) { + return std::shared_ptr
(ctx.GetFormatTable()); + } + PAIMON_RETURN_NOT_OK(CheckVersionManagementImplemented(ctx.GetCatalog())); + std::shared_ptr specific_fs = ctx.GetSpecificFileSystem(); + if (specific_fs == nullptr && ctx.GetFileSystemSchemeToIdentifierMap().empty() && + ctx.GetCatalog() != nullptr) { + if (!ctx.GetIdentifier()) { + return Status::Invalid("a catalog write requires a table identifier"); + } + // A catalog issuing per-table temporary credentials only hands them out through + // GetTableFileSystem, so the write uses this table's. + PAIMON_ASSIGN_OR_RAISE(specific_fs, + ctx.GetCatalog()->GetTableFileSystem(ctx.GetIdentifier().value())); + } + PAIMON_ASSIGN_OR_RAISE(CoreOptions tmp_options, + CoreOptions::FromMap(ctx.GetOptions(), specific_fs, + ctx.GetFileSystemSchemeToIdentifierMap())); + std::string branch = ctx.GetBranch(); + PAIMON_ASSIGN_OR_RAISE(std::optional catalog_identifier, + CatalogUtils::BranchIdentifier(ctx.GetIdentifier(), branch)); + // Only the main branch may have a current schema the catalog alone holds: a branch publishes + // its own under `branch/branch-`, which is where the load below reads it instead. + PAIMON_ASSIGN_OR_RAISE( + std::optional catalog_table_schema, + CatalogMainBranchSchemaJson(ctx.GetCatalog(), catalog_identifier, branch, "write")); + return ManagedTableOf(tmp_options.GetFileSystem(), specific_fs, ctx.GetRootPath(), branch, + ctx.GetOptions(), catalog_table_schema, ctx.GetIdentifier(), /*uuid=*/"", + ctx.GetFileSystemSchemeToIdentifierMap()); +} + +/// The table a commit context names: the format table the builder was handed, or the table the +/// schema under the context's root path says is there. +Result> FileStoreTableFactory::Create(const CommitContext& ctx) { + if (ctx.GetFormatTable() != nullptr) { + return std::shared_ptr
(ctx.GetFormatTable()); + } + PAIMON_RETURN_NOT_OK(CheckVersionManagementImplemented(ctx.GetCatalog())); + // Use this table's catalog credentials unless the caller supplied a file system: a catalog + // issuing per-table temporary ones only hands them out through GetTableFileSystem. + std::shared_ptr specific_fs = ctx.GetSpecificFileSystem(); + if (specific_fs == nullptr && ctx.GetCatalog() != nullptr) { + if (!ctx.GetIdentifier()) { + return Status::Invalid("a catalog commit requires a table identifier"); + } + PAIMON_ASSIGN_OR_RAISE(specific_fs, + ctx.GetCatalog()->GetTableFileSystem(ctx.GetIdentifier().value())); + } + PAIMON_ASSIGN_OR_RAISE(CoreOptions tmp_options, + CoreOptions::FromMap(ctx.GetOptions(), specific_fs)); + const std::string& root_path = ctx.GetRootPath(); + // The branch the context was built for, already validated by `CommitContextBuilder`. + PAIMON_ASSIGN_OR_RAISE(std::string branch, BranchManager::ResolveBranch( + ctx.GetIdentifier(), ctx.GetOptions(), + /*explicit_branch=*/std::nullopt, "commit")); + PAIMON_ASSIGN_OR_RAISE(std::optional catalog_identifier, + CatalogUtils::BranchIdentifier(ctx.GetIdentifier(), branch)); + // Only the main branch may have a current schema the catalog alone holds: a branch publishes + // its own under `branch/branch-`, which is where the load below reads it instead. + PAIMON_ASSIGN_OR_RAISE( + std::optional catalog_table_schema, + CatalogMainBranchSchemaJson(ctx.GetCatalog(), catalog_identifier, branch, "commit")); + // The managed path below reads the commit's branch, so this reads the same one: the two must + // not dispatch on different schemas. + return ManagedTableOf(tmp_options.GetFileSystem(), specific_fs, root_path, branch, + ctx.GetOptions(), catalog_table_schema, ctx.GetIdentifier(), + ctx.GetTableId().value_or("")); +} + +} // namespace paimon diff --git a/src/paimon/core/table/file_store_table_factory.h b/src/paimon/core/table/file_store_table_factory.h new file mode 100644 index 000000000..175834ff6 --- /dev/null +++ b/src/paimon/core/table/file_store_table_factory.h @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include + +#include "paimon/result.h" + +namespace paimon { + +class CommitContext; +class ReadContext; +class ScanContext; +class Table; +class WriteContext; + +/// The table a context names, Java Paimon's `FileStoreTableFactory`: the format table the builder +/// was handed, a system table behind a `T$system` path, or the table the schema under the +/// context's path says is there. The four operators' static `Create` entries all dispatch on the +/// same question, so one place answers it and they cannot disagree. +class FileStoreTableFactory { + public: + FileStoreTableFactory() = delete; + ~FileStoreTableFactory() = delete; + + static Result> Create(const ScanContext& context); + + static Result> Create(const ReadContext& context); + + static Result> Create(const WriteContext& context); + + static Result> Create(const CommitContext& context); +}; + +} // namespace paimon diff --git a/src/paimon/core/utils/context_utils.h b/src/paimon/core/utils/context_utils.h new file mode 100644 index 000000000..57e337904 --- /dev/null +++ b/src/paimon/core/utils/context_utils.h @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "fmt/format.h" +#include "paimon/status.h" + +namespace paimon { + +/// The checks every kind of table's operator entry points share. +class ContextUtils { + public: + ContextUtils() = delete; + ~ContextUtils() = delete; + + /// The checks every operator entry point opens with: a context, a memory pool and an executor + /// must all be there for the operator to be built at all, so a null one is refused rather than + /// dereferenced. `kind` names the operator the first message speaks of, which is all that sets + /// the entry points' messages apart. + template + static Status CheckOperatorContext(const ContextPointer& context, const char* kind) { + if (context == nullptr) { + return Status::Invalid(fmt::format("{} context is null pointer", kind)); + } + if (context->GetMemoryPool() == nullptr) { + return Status::Invalid("memory pool is null pointer"); + } + if (context->GetExecutor() == nullptr) { + return Status::Invalid("executor is null pointer"); + } + return Status::OK(); + } +}; + +} // namespace paimon From f4437a7dca367d0908f93b9b7ecfe20c02faebad Mon Sep 17 00:00:00 2001 From: "yonghao.fyh" Date: Tue, 27 Oct 2026 16:10:48 +0800 Subject: [PATCH 16/17] fix --- benchmark/benchmark_suite.cpp | 2 +- docs/source/user_guide/catalog.rst | 20 +-- docs/source/user_guide/format_table.rst | 8 +- examples/read_write_demo.cpp | 2 +- include/paimon/catalog/catalog.h | 11 +- include/paimon/catalog/table.h | 16 +-- include/paimon/file_store_commit.h | 4 +- include/paimon/file_store_write.h | 4 +- include/paimon/table/source/table_read.h | 4 +- include/paimon/table/source/table_scan.h | 4 +- src/paimon/core/catalog/catalog.cpp | 15 ++- .../core/catalog/file_system_catalog.cpp | 15 ++- .../core/catalog/file_system_catalog_test.cpp | 16 ++- ...eted_primary_key_index_maintainer_test.cpp | 4 +- .../core/operation/file_store_write_test.cpp | 5 +- .../core/operation/internal_read_context.cpp | 30 +++-- .../core/operation/internal_read_context.h | 41 +++--- .../core/table/append_only_file_store_table.h | 2 +- src/paimon/core/table/file_store.h | 114 ++++++++--------- src/paimon/core/table/file_store_table.cpp | 6 +- src/paimon/core/table/file_store_table.h | 120 +++++++++--------- .../core/table/file_store_table_factory.cpp | 18 +-- .../core/table/file_store_table_factory.h | 8 +- src/paimon/core/table/format/format_table.cpp | 30 +++-- .../core/table/format/format_table_test.cpp | 29 +++++ .../core/table/primary_key_file_store_table.h | 2 +- .../core/table/source/table_read_test.cpp | 10 +- .../table/system/audit_log_system_table.cpp | 15 ++- .../table/system/audit_log_system_table.h | 7 +- .../core/table/system/binlog_system_table.cpp | 8 +- .../core/table/system/binlog_system_table.h | 1 - .../table/system/global_system_tables.cpp | 70 +++++----- .../core/table/system/global_system_tables.h | 25 ++-- .../table/system/in_memory_system_table.cpp | 4 +- .../table/system/in_memory_system_table.h | 3 +- .../table/system/metadata_system_tables.cpp | 49 ++----- .../table/system/metadata_system_tables.h | 8 -- .../system/read_optimized_system_table.cpp | 8 +- .../system/read_optimized_system_table.h | 1 - src/paimon/core/table/system/system_table.cpp | 19 ++- src/paimon/core/table/system/system_table.h | 48 ++++--- .../core/table/system/system_table_test.cpp | 63 +++++++-- src/paimon/core/table/table_test.cpp | 26 +++- src/paimon/rest/rest_catalog.cpp | 23 ++-- src/paimon/rest/rest_catalog_test.cpp | 38 +++++- src/paimon/testing/mock/mock_catalog.h | 4 +- test/inte/read_inte_test.cpp | 29 +++-- test/inte/realtime_write_inte_test.cpp | 4 +- 48 files changed, 571 insertions(+), 422 deletions(-) diff --git a/benchmark/benchmark_suite.cpp b/benchmark/benchmark_suite.cpp index 0a48a9173..6ec44f0d2 100644 --- a/benchmark/benchmark_suite.cpp +++ b/benchmark/benchmark_suite.cpp @@ -368,7 +368,7 @@ Status EnsureTable(const std::string& root_path, const std::string& db_name, const std::shared_ptr& schema, const std::vector& primary_keys = {}) { PAIMON_ASSIGN_OR_RAISE( - std::unique_ptr catalog, + std::shared_ptr catalog, AddContext(paimon::Catalog::Create(root_path, options), "create catalog")); PAIMON_RETURN_NOT_OK( AddContext(catalog->CreateDatabase(db_name, options, true), "create database")); diff --git a/docs/source/user_guide/catalog.rst b/docs/source/user_guide/catalog.rst index d881968c4..599b4ffce 100644 --- a/docs/source/user_guide/catalog.rst +++ b/docs/source/user_guide/catalog.rst @@ -83,7 +83,7 @@ registered on the REST server. The catalog is configured through the {"token.provider", "bear"}, {"token", ""}, }; - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr catalog, + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr catalog, paimon::Catalog::Create(/*root_path=*/"my_instance", options)); For DLF, configure one credential source. Static AK/SK credentials, an optional @@ -122,15 +122,12 @@ without an atomic snapshot-file rename: .. code-block:: cpp - // Writers and committers share ownership of the catalog. - std::shared_ptr shared_catalog(std::move(catalog)); - // Load the table's identity before preparing changes. PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table, - shared_catalog->GetTable(paimon::Identifier("db", "tbl"))); + catalog->GetTable(paimon::Identifier("db", "tbl"))); paimon::CommitContextBuilder builder(table_path, "commit-user"); - builder.WithCatalog(shared_catalog, paimon::Identifier("db", "tbl")); + builder.WithCatalog(catalog, paimon::Identifier("db", "tbl")); // CatalogUuid() stays null when the server supplies no id. if (std::optional table_uuid = table->CatalogUuid()) { builder.WithTableId(table_uuid.value()); @@ -147,7 +144,7 @@ offsets when a real-time writer is created: .. code-block:: cpp paimon::WriteContextBuilder write_builder(table_path, "commit-user"); - write_builder.WithCatalog(shared_catalog, paimon::Identifier("db", "tbl")); + write_builder.WithCatalog(catalog, paimon::Identifier("db", "tbl")); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr write_context, write_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr writer, @@ -183,7 +180,7 @@ returns credentials that are still valid and reloads them before they expire. "oss://bucket/tbl", options)); // MyFileSystem merges provider->GetCredentials() over its own options per access. auto my_fs = std::make_shared(provider); - write_builder.WithCatalog(shared_catalog, paimon::Identifier("db", "tbl")) + write_builder.WithCatalog(catalog, paimon::Identifier("db", "tbl")) .WithFileSystem(my_fs); The snapshot being committed carries a uuid generated on the client, and the @@ -262,18 +259,15 @@ for that table's file system and passing it in through ``WithFileSystem``: .. code-block:: cpp - // Readers and scanners share ownership of the catalog. - std::shared_ptr shared_catalog(std::move(catalog)); - paimon::ScanContextBuilder scan_builder(table_path); - scan_builder.WithCatalog(shared_catalog, paimon::Identifier("db", "tbl")); + scan_builder.WithCatalog(catalog, paimon::Identifier("db", "tbl")); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan_context, scan_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan, paimon::TableScan::Create(std::move(scan_context))); paimon::ReadContextBuilder read_builder(table_path); - read_builder.WithCatalog(shared_catalog, paimon::Identifier("db", "tbl")); + read_builder.WithCatalog(catalog, paimon::Identifier("db", "tbl")); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, read_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read, diff --git a/docs/source/user_guide/format_table.rst b/docs/source/user_guide/format_table.rst index 4be21a1dd..7ee90976d 100644 --- a/docs/source/user_guide/format_table.rst +++ b/docs/source/user_guide/format_table.rst @@ -53,9 +53,9 @@ That path needs a schema under the table's own location, which is what every tab catalog serves has. A table loaded from a catalog that keeps the schema elsewhere, such as the REST catalog, has none: nothing under the location says that it is a format table, nor that ``schema`` and ``branch`` below it are data rather than metadata. Such a table is read and written -by handing the loaded ``FormatTable`` to the context instead of a path - every context builder has -a constructor taking one - which is what Java does through ``FormatTable.newReadBuilder()`` and -``newBatchWriteBuilder()``: +by handing the loaded ``FormatTable`` to the context builder instead of a path - every builder has +a constructor taking one - and calling the table's own operator, which is what Java does through +``FormatTable.newReadBuilder()`` and ``newBatchWriteBuilder()``: .. code-block:: cpp @@ -68,7 +68,7 @@ a constructor taking one - which is what Java does through ``FormatTable.newRead PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan_context, scan_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan, - paimon::TableScan::Create(std::move(scan_context))); + table->NewScan(std::move(scan_context))); The table already carries its schema and the file system it was loaded through, so a context built from one refuses ``SetTableSchema()``, ``WithFileSystem()``, diff --git a/examples/read_write_demo.cpp b/examples/read_write_demo.cpp index df46fcc54..e911bedb1 100644 --- a/examples/read_write_demo.cpp +++ b/examples/read_write_demo.cpp @@ -61,7 +61,7 @@ paimon::Status Run(const std::string& root_path, const std::string& db_name, {paimon::Options::FILE_SYSTEM, "local"}}; // create table - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr catalog, + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr catalog, paimon::Catalog::Create(root_path, options)); PAIMON_RETURN_NOT_OK(catalog->CreateDatabase(db_name, options, /*ignore_if_exists=*/false)); arrow::FieldVector fields = { diff --git a/include/paimon/catalog/catalog.h b/include/paimon/catalog/catalog.h index 033e7a822..3898a1967 100644 --- a/include/paimon/catalog/catalog.h +++ b/include/paimon/catalog/catalog.h @@ -68,8 +68,11 @@ class PAIMON_EXPORT Catalog { /// schemes, including the file systems it builds from per-table data /// tokens, keeps routing each scheme to its backend. Ignored when /// `file_system` is supplied, which serves every scheme itself. - /// @return A result containing a unique pointer to a `Catalog` instance, or an error status. - static Result> Create( + /// @return A result containing a shared pointer to a `Catalog` instance, or an error status. + /// @note Shared rather than unique ownership: a global system table of the `sys` database + /// builds its rows lazily through the catalog it was handed, so the table keeps the + /// catalog alive. A catalog built by this factory always satisfies that. + static Result> Create( const std::string& root_path, const std::map& options, const std::shared_ptr& file_system = nullptr, const std::map& fs_scheme_to_identifier_map = {}); @@ -154,7 +157,9 @@ class PAIMON_EXPORT Catalog { /// `NewRead()` serve any of them, `NewWrite()` and `NewCommit()` the writable ones. /// /// A global system table of the "sys" database reads its rows lazily through the catalog it - /// came from, so a catalog that serves one must be owned by a `std::shared_ptr`. + /// came from, so the table keeps shared ownership of that catalog. `Catalog::Create()` hands + /// back a `std::shared_ptr`, which satisfies this. A catalog held some other way, such as + /// through a `std::unique_ptr`, cannot be kept and is refused by name. /// /// @param identifier Identifier of the table to get. /// @return A result containing the table, or an error status. diff --git a/include/paimon/catalog/table.h b/include/paimon/catalog/table.h index 3364f6eff..9164623c0 100644 --- a/include/paimon/catalog/table.h +++ b/include/paimon/catalog/table.h @@ -45,10 +45,10 @@ class WriteContext; /// A table provides basic abstraction for table type. /// /// The state every kind of table carries - file system, root path, schema and identifier - lives -/// here, and each kind fills the operations in. The `NewXxx` operators below are the table-first -/// entry points: `TableScan::Create()` and friends load the table a context names and hand the -/// context to them. What a context asks for that the table cannot honour is refused by name rather -/// than silently dropped. +/// here, and each kind fills in the operations. The `NewXxx` operators below are the table-first +/// entry points: `TableScan::Create()` and its friends load the table a context names and hand the +/// context to it. A table refuses by name whatever a context asks for that it cannot honour, +/// rather than dropping it silently. class PAIMON_EXPORT Table { public: virtual ~Table() = default; @@ -65,8 +65,8 @@ class PAIMON_EXPORT Table { return root_path_; } - /// A name to identify this table: a managed or format table answers its identifier's object - /// name, a system table its own kind. + /// A name to identify this table. A managed or format table answers the object name of its + /// identifier, a system table its own name. virtual std::string Name() const = 0; /// Full name of the table: `database.tableName` for a managed or format table, the same name @@ -90,9 +90,9 @@ class PAIMON_EXPORT Table { return schema_; } - /// Copies this table with `dynamic_options` on top of the options it carries: an option given + /// Copies this table with `dynamic_options` on top of the options it carries. An option given /// at the call wins over the one the schema stored, the precedence every context builder - /// promises. `type` is not overridable - it is structural and read from the schema alone. + /// promises. `type` is not overridable: it is structural and read from the schema alone. /// /// @param dynamic_options Options given at the call. /// @return A result containing the copied table, or the error copying it reports. diff --git a/include/paimon/file_store_commit.h b/include/paimon/file_store_commit.h index 581cadf2d..8a6eaad5c 100644 --- a/include/paimon/file_store_commit.h +++ b/include/paimon/file_store_commit.h @@ -50,8 +50,8 @@ class CommitMessage; /// manages versions. `UseRESTCatalogCommit()` only prepares a request for the caller to send. class PAIMON_EXPORT FileStoreCommit { public: - /// Create an instance of `FileStoreCommit`: the table the context names is loaded first, then - /// the context is handed to that table's `NewCommit`. + /// Create an instance of `FileStoreCommit`. A context that already carries a loaded table goes + /// straight to that table's `NewCommit`; otherwise the table the context names is loaded first. /// /// @param context A unique pointer to the `CommitContext` used for commit operations. /// diff --git a/include/paimon/file_store_write.h b/include/paimon/file_store_write.h index 5f74d6536..0778e5bb7 100644 --- a/include/paimon/file_store_write.h +++ b/include/paimon/file_store_write.h @@ -42,8 +42,8 @@ class WriteContext; /// Interface for write operations in a file store. class PAIMON_EXPORT FileStoreWrite { public: - /// Create an instance of `FileStoreWrite`: the table the context names is loaded first, then - /// the context is handed to that table's `NewWrite`. + /// Create an instance of `FileStoreWrite`. A context that already carries a loaded table goes + /// straight to that table's `NewWrite`; otherwise the table the context names is loaded first. /// /// @param context A unique pointer to the `WriteContext` used for write operations. /// diff --git a/include/paimon/table/source/table_read.h b/include/paimon/table/source/table_read.h index 2bfd84fff..5f8303fe3 100644 --- a/include/paimon/table/source/table_read.h +++ b/include/paimon/table/source/table_read.h @@ -42,8 +42,8 @@ class PAIMON_EXPORT TableRead { public: virtual ~TableRead() = default; - /// Create an instance of `TableRead`: the table the context names is loaded first, then the - /// context is handed to that table's `NewRead`. + /// Create an instance of `TableRead`. A context that already carries a loaded table goes + /// straight to that table's `NewRead`; otherwise the table the context names is loaded first. /// /// @param context A unique pointer to the `ReadContext` used for read operations. /// @return A Result containing a unique pointer to the `TableRead` instance. diff --git a/include/paimon/table/source/table_scan.h b/include/paimon/table/source/table_scan.h index 39f0453c8..4944dafd2 100644 --- a/include/paimon/table/source/table_scan.h +++ b/include/paimon/table/source/table_scan.h @@ -36,8 +36,8 @@ class ScanContext; /// A scanner interface for reading table's meta and create a plan. class PAIMON_EXPORT TableScan { public: - /// Create an instance of `TableScan`: the table the context names is loaded first, then the - /// context is handed to that table's `NewScan`. + /// Create an instance of `TableScan`. A context that already carries a loaded table goes + /// straight to that table's `NewScan`; otherwise the table the context names is loaded first. /// /// @param context A unique pointer to the `ScanContext` used for scan operations. /// @return A Result containing a unique pointer to the `TableScan` instance. diff --git a/src/paimon/core/catalog/catalog.cpp b/src/paimon/core/catalog/catalog.cpp index 318125ec6..b14a53eab 100644 --- a/src/paimon/core/catalog/catalog.cpp +++ b/src/paimon/core/catalog/catalog.cpp @@ -40,7 +40,7 @@ const char Catalog::SYSTEM_TABLE_SPLITTER[] = "$"; const char Catalog::DB_SUFFIX[] = ".db"; const char Catalog::DB_LOCATION_PROP[] = "location"; -Result> Catalog::Create( +Result> Catalog::Create( const std::string& root_path, const std::map& options, const std::shared_ptr& file_system, const std::map& fs_scheme_to_identifier_map) { @@ -53,8 +53,14 @@ Result> Catalog::Create( } if (metastore == "rest") { #ifdef PAIMON_ENABLE_REST - return RestCatalog::Create(root_path, options, file_system, RestHttpClient::Config(), - fs_scheme_to_identifier_map); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr rest_catalog, + RestCatalog::Create(root_path, options, file_system, RestHttpClient::Config(), + fs_scheme_to_identifier_map)); + // Built from the derived type so that the `enable_shared_from_this` base a + // `RestCatalog` carries is initialized: a `sys` table keeps its catalog alive through + // `weak_from_this()`, which stays empty when the pointer is widened first. + return std::shared_ptr(std::move(rest_catalog)); #else return Status::NotImplemented( "the rest catalog requires building paimon with PAIMON_ENABLE_REST=ON"); @@ -65,7 +71,8 @@ Result> Catalog::Create( } PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, CoreOptions::FromMap(options, file_system, fs_scheme_to_identifier_map)); - return std::make_unique(core_options.GetFileSystem(), root_path, options); + return std::shared_ptr( + std::make_unique(core_options.GetFileSystem(), root_path, options)); } bool Catalog::SupportsVersionManagement() const { diff --git a/src/paimon/core/catalog/file_system_catalog.cpp b/src/paimon/core/catalog/file_system_catalog.cpp index fdcddf20e..d061acbf2 100644 --- a/src/paimon/core/catalog/file_system_catalog.cpp +++ b/src/paimon/core/catalog/file_system_catalog.cpp @@ -287,16 +287,15 @@ Result> FileSystemCatalog::LoadTableSchema( if (!supported) { return Status::NotExist(fmt::format("{} not exist", identifier.ToString())); } - // The table built here is transient - its schema is read within this call and nothing - // outlives it - so its context keeps no shared ownership of the catalog. A catalog that - // hands a global system table back out (`RestCatalog::GetTable`) retains it instead. + // The table built here is transient: only its schema is read and nothing outlives the + // call. It builds no rows, so the context carries no catalog. GlobalSystemTableContext context; - context.catalog = this; context.fs = fs_; context.warehouse = warehouse_; context.catalog_options = catalog_options_; - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr system_table, - GlobalSystemTableLoader::Load(identifier.GetObjectName(), context)); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr system_table, + GlobalSystemTableLoader::LoadForSchema(identifier.GetObjectName(), context)); return system_table->LatestSchema(); } // The system table suffix is dropped when the data table identifier is rebuilt below, so the @@ -351,7 +350,9 @@ Result> FileSystemCatalog::GetTable(const Identifier& ide // The branch the schema above was read on, so the table's managers read the same one. PAIMON_ASSIGN_OR_RAISE(std::string branch, identifier.GetBranchNameOrDefault()); return FileStoreTable::Create(fs_, table_path, checked_pointer_cast(*latest_schema), - identifier, /*uuid=*/"", branch); + identifier, /*uuid=*/"", branch, /*schema_manager=*/nullptr, + /*snapshot_manager=*/nullptr, /*fs_scheme_map=*/{}, + /*dynamic_options=*/{}); } Result> FileSystemCatalog::LoadSystemTable( diff --git a/src/paimon/core/catalog/file_system_catalog_test.cpp b/src/paimon/core/catalog/file_system_catalog_test.cpp index f31c0391d..294b7e9b9 100644 --- a/src/paimon/core/catalog/file_system_catalog_test.cpp +++ b/src/paimon/core/catalog/file_system_catalog_test.cpp @@ -231,11 +231,12 @@ TEST(FileSystemCatalogTest, TestOptionsSystemTableCatalog) { ASSERT_OK_AND_ASSIGN(auto system_table, catalog.GetTable(options_identifier)); // A system table comes back as the `SystemTable` it is: it answers for itself with its own - // read-only schema and the root path of the data table it reads. + // read-only schema and the root path of the data table it reads. Its name carries the data + // table it belongs to. ASSERT_TRUE(std::dynamic_pointer_cast(system_table) != nullptr); ASSERT_TRUE(std::dynamic_pointer_cast(system_table) == nullptr); - ASSERT_EQ("options", system_table->Name()); - ASSERT_EQ("options", system_table->FullName()); + ASSERT_EQ("tbl1$options", system_table->Name()); + ASSERT_EQ("tbl1$options", system_table->FullName()); ASSERT_EQ(system_table->GetRootPath(), table_path); ASSERT_EQ((std::vector{"key", "value"}), system_table->LatestSchema()->FieldNames()); @@ -281,7 +282,7 @@ TEST(FileSystemCatalogTest, TestGetTableServesASystemTable) { ASSERT_OK_AND_ASSIGN(auto system_table, catalog.GetTable(Identifier("db1", "tbl1$options"))); ASSERT_TRUE(std::dynamic_pointer_cast(system_table) != nullptr); - ASSERT_EQ("options", system_table->Name()); + ASSERT_EQ("tbl1$options", system_table->Name()); // The scan and the read work from a context of the caller's, like every other table. ScanContextBuilder scan_builder(system_table->GetRootPath()); @@ -310,13 +311,14 @@ TEST(FileSystemCatalogTest, TestGetTableServesASystemTable) { write_builder.SetOptions(options); ASSERT_OK_AND_ASSIGN(auto write_context, write_builder.Finish()); ASSERT_NOK_WITH_MSG(system_table->NewWrite(std::move(write_context)), - "options system table is read-only: writing is not supported"); + "tbl1$options system table is read-only: writing is not supported"); CommitContextBuilder commit_builder(system_table->GetRootPath(), "commit_user"); commit_builder.SetOptions(options); ASSERT_OK_AND_ASSIGN(auto commit_context, commit_builder.Finish()); - ASSERT_NOK_WITH_MSG(system_table->NewCommit(std::move(commit_context)), - "options system table is read-only: committing for it is not supported"); + ASSERT_NOK_WITH_MSG( + system_table->NewCommit(std::move(commit_context)), + "tbl1$options system table is read-only: committing for it is not supported"); } TEST(FileSystemCatalogTest, TestBranchTableLoadsBranchSchema) { diff --git a/src/paimon/core/index/pk/bucketed_primary_key_index_maintainer_test.cpp b/src/paimon/core/index/pk/bucketed_primary_key_index_maintainer_test.cpp index 81803ed90..85690b25f 100644 --- a/src/paimon/core/index/pk/bucketed_primary_key_index_maintainer_test.cpp +++ b/src/paimon/core/index/pk/bucketed_primary_key_index_maintainer_test.cpp @@ -298,7 +298,7 @@ class BucketedPrimaryKeyIndexMaintainerTest : public ::testing::TestWithParam catalog, + ASSERT_OK_AND_ASSIGN(std::shared_ptr catalog, Catalog::Create(directory_->Str(), {})); ASSERT_OK(catalog->CreateDatabase("db", {}, /*ignore_if_exists=*/false)); ASSERT_OK(catalog->CreateTable(Identifier("db", "table"), &c_schema, @@ -373,7 +373,7 @@ TEST_P(BucketedPrimaryKeyIndexMaintainerTest, BuildsIndexForColumnNamedLikeInter schema_ = arrow::schema({arrow::field("id", arrow::int32(), /*nullable=*/false), arrow::field(kColumnName, arrow::utf8(), /*nullable=*/false)}); options_[Options::PK_BTREE_INDEX_COLUMNS] = kColumnName; - ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, Catalog::Create(directory_->Str(), {})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr catalog, Catalog::Create(directory_->Str(), {})); for (bool spillable : {false, true}) { SCOPED_TRACE(spillable); const std::string table_name = spillable ? "row_id_spill" : "row_id_memory"; diff --git a/src/paimon/core/operation/file_store_write_test.cpp b/src/paimon/core/operation/file_store_write_test.cpp index 520e850e5..ffee33161 100644 --- a/src/paimon/core/operation/file_store_write_test.cpp +++ b/src/paimon/core/operation/file_store_write_test.cpp @@ -170,14 +170,13 @@ TEST(FileStoreWriteTest, TestCreateAppendTable) { ASSERT_OK(catalog->CreateTable(Identifier("foo", "bar"), &schema, /*partition_keys=*/{"f0", "f3"}, /*primary_keys=*/{}, /*options=*/{}, /*ignore_if_exists=*/false)); - std::shared_ptr shared_catalog(std::move(catalog)); - ASSERT_FALSE(shared_catalog->SupportsVersionManagement()); + ASSERT_FALSE(catalog->SupportsVersionManagement()); for (bool use_catalog : {false, true}) { SCOPED_TRACE(use_catalog); WriteContextBuilder context_builder(PathUtil::JoinPath(dir->Str(), "foo.db/bar"), "commit_user_1"); if (use_catalog) { - context_builder.WithCatalog(shared_catalog, Identifier("foo", "bar")); + context_builder.WithCatalog(catalog, Identifier("foo", "bar")); } ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, context_builder.Finish()); ASSERT_OK_AND_ASSIGN(std::unique_ptr file_store_write, diff --git a/src/paimon/core/operation/internal_read_context.cpp b/src/paimon/core/operation/internal_read_context.cpp index 8866eae9f..3ee0eb755 100644 --- a/src/paimon/core/operation/internal_read_context.cpp +++ b/src/paimon/core/operation/internal_read_context.cpp @@ -277,17 +277,24 @@ std::optional InternalReadContext::TryResolveSpecialFieldByName( return std::nullopt; } -Result> InternalReadContext::Create( +Result> InternalReadContext::CreateForTable( const std::shared_ptr& context, const std::shared_ptr& table_schema, - CoreOptions core_options, const std::string* path) { + CoreOptions core_options, const std::string& root_path) { core_options.WithCache(context->GetCache()); - return CreateFromCoreOptions(context, table_schema, std::move(core_options), path); + return CreateFromCoreOptions(context, table_schema, std::move(core_options), root_path); } Result> InternalReadContext::Create( + const std::shared_ptr& context, const std::shared_ptr& table_schema, + const std::map& options) { + return CreateFromOptions(context, table_schema, options, /*file_system=*/nullptr, + /*path=*/std::nullopt); +} + +Result> InternalReadContext::CreateFromOptions( const std::shared_ptr& context, const std::shared_ptr& table_schema, const std::map& options, - const std::shared_ptr& file_system, const std::string* path) { + const std::shared_ptr& file_system, std::optional path) { // The file system the caller named, or null for the read context's own, which callers // without a table hand over. The scheme map the context carries does not take part: a file // system handed over wins and one missing falls to the resolving one, the same way a scan @@ -297,7 +304,7 @@ Result> InternalReadContext::Create( CoreOptions::FromMap( options, file_system != nullptr ? file_system : context->GetSpecificFileSystem())); core_options.WithCache(context->GetCache()); - return CreateFromCoreOptions(context, table_schema, std::move(core_options), path); + return CreateFromCoreOptions(context, table_schema, std::move(core_options), std::move(path)); } Result> InternalReadContext::CreateForFallbackBranch( @@ -341,12 +348,12 @@ Result> InternalReadContext::CreateForFallb } else { options.erase(Options::BRANCH); } - return Create(context, table_schema, options, file_system, &root_path); + return CreateFromOptions(context, table_schema, options, file_system, root_path); } Result> InternalReadContext::CreateFromCoreOptions( const std::shared_ptr& context, const std::shared_ptr& table_schema, - CoreOptions core_options, const std::string* path) { + CoreOptions core_options, std::optional path) { if (context->HasVectorOrFullTextSearch()) { if (!core_options.FileIndexReadEnabled()) { return Status::Invalid( @@ -458,8 +465,7 @@ Result> InternalReadContext::CreateFromCore } std::shared_ptr arrow_pool = GetArrowPool(context->GetMemoryPool()); auto internal_context = std::unique_ptr(new InternalReadContext( - context, table_schema, read_schema, core_options, arrow_pool, - path != nullptr ? std::optional(*path) : std::nullopt)); + context, table_schema, read_schema, core_options, std::move(path), arrow_pool)); PAIMON_RETURN_NOT_OK(internal_context->BindIndexScores()); return internal_context; } @@ -539,8 +545,8 @@ InternalReadContext::InternalReadContext(const std::shared_ptr& rea const std::shared_ptr& table_schema, const std::shared_ptr& read_schema, const CoreOptions& options, - const std::shared_ptr& arrow_pool, - std::optional path) + std::optional path, + const std::shared_ptr& arrow_pool) : read_context_(read_context), table_schema_(table_schema), read_schema_(read_schema), @@ -560,7 +566,7 @@ Result> InternalReadContext::CreateWithSche // The new read_schema is the minimal column set for COUNT(*). return std::shared_ptr( new InternalReadContext(original->read_context_, original->table_schema_, new_read_schema, - original->options_, original->arrow_pool_, original->path_)); + original->options_, original->path_, original->arrow_pool_)); } } // namespace paimon diff --git a/src/paimon/core/operation/internal_read_context.h b/src/paimon/core/operation/internal_read_context.h index 43783e950..9e5d91ce0 100644 --- a/src/paimon/core/operation/internal_read_context.h +++ b/src/paimon/core/operation/internal_read_context.h @@ -58,24 +58,23 @@ class InternalReadContext { std::string field_name; }; - /// Builds the context of a read a table serves: the options are the table's own, already - /// parsed, and the cache the context carries is armed on them here. `path` names the root path - /// the table was opened with, which the read reads from in place of the context's own. - static Result> Create( + /// Builds the context of the read a table serves: the options are the table's own, already + /// parsed, and the cache the context carries is armed on them here. + /// @param root_path The root path the table was opened with. The read reads from it in place of + /// the path the context names. + static Result> CreateForTable( const std::shared_ptr& read_context, const std::shared_ptr& table_schema, CoreOptions core_options, - const std::string* path = nullptr); + const std::string& root_path); /// Builds the context of a read whose options are composed on the spot rather than the table's - /// own - the fallback branch of a read, whose schema belongs to another branch, and the reads a - /// write or a lookup serve inside themselves. The read's own file system is `file_system`, or - /// the context's when null; the scheme map the context carries does not take part. + /// own: the reads a write or a lookup serve inside themselves. Such a read serves the file + /// system the context resolved and reads from the path the context names. A fallback branch + /// reads through `CreateForFallbackBranch` instead, whose schema belongs to another branch. static Result> Create( const std::shared_ptr& read_context, const std::shared_ptr& table_schema, - const std::map& options, - const std::shared_ptr& file_system = nullptr, - const std::string* path = nullptr); + const std::map& options); /// The read context of a fallback branch, whose schema is not the table's own: it is read /// through a manager of that branch, and the read serves the options that schema was loaded @@ -195,16 +194,24 @@ class InternalReadContext { InternalReadContext(const std::shared_ptr& read_context, const std::shared_ptr& table_schema, const std::shared_ptr& read_schema, - const CoreOptions& options, - const std::shared_ptr& arrow_pool, - std::optional path = std::nullopt); + const CoreOptions& options, std::optional path, + const std::shared_ptr& arrow_pool); + + /// The tail of a read whose options are a map: they resolve to `CoreOptions` against + /// `file_system`, or against the context's own when it is null, and then go where + /// `CreateFromCoreOptions` takes them. + static Result> CreateFromOptions( + const std::shared_ptr& read_context, + const std::shared_ptr& table_schema, + const std::map& options, + const std::shared_ptr& file_system, std::optional path); /// The tail both `Create` entries share: the options arrive parsed and cached, and the /// validations and the read schema below are the same whichever way they were resolved. static Result> CreateFromCoreOptions( const std::shared_ptr& read_context, const std::shared_ptr& table_schema, CoreOptions core_options, - const std::string* path); + std::optional path); static std::optional TryResolveSpecialFieldById(int32_t field_id, const CoreOptions& core_options, @@ -224,8 +231,8 @@ class InternalReadContext { std::shared_ptr arrow_pool_; CoreOptions options_; std::map index_score_bindings_; - /// The root path a table reads through, or null for the read context's own, which callers - /// without a table hand over. + /// The root path a table reads through, or empty for the read context's own, which a caller + /// without a table leaves it as. std::optional path_; }; diff --git a/src/paimon/core/table/append_only_file_store_table.h b/src/paimon/core/table/append_only_file_store_table.h index 9b4af46ae..1f94d580a 100644 --- a/src/paimon/core/table/append_only_file_store_table.h +++ b/src/paimon/core/table/append_only_file_store_table.h @@ -43,7 +43,7 @@ class AppendOnlyFileStoreTable : public FileStoreTable { /// The constructor `FileStoreTable::Create` builds an append-only table through: the store /// carries every state, and the base class holds it. AppendOnlyFileStoreTable(const std::shared_ptr& store, const Identifier& identifier, - const std::string& uuid = ""); + const std::string& uuid); private: Result> CreateWrite( diff --git a/src/paimon/core/table/file_store.h b/src/paimon/core/table/file_store.h index e343ec2c4..339dd3810 100644 --- a/src/paimon/core/table/file_store.h +++ b/src/paimon/core/table/file_store.h @@ -46,51 +46,47 @@ class ManifestFile; class ManifestList; class TableSchema; -/// The storage a managed table is served through: the file system, root path, schema, branch, -/// options and managers it was loaded with - the state Java Paimon's `AbstractFileStore` carries, -/// parsed once here rather than per operator. `FileStoreTable` hands the state it was loaded with -/// to one of these and asks it back for the pieces every operator opens with. +/// Holds the storage a managed table reads and writes through: the file system, root path, +/// schema, branch, options and managers. Java Paimon carries the same state in `AbstractFileStore`. +/// `FileStoreTable` builds one when it loads and asks it for the pieces each operator opens with. /// -/// Every operator serves the options the table was loaded with, so the factories below take the -/// options of the call: a cache is not part of the table's, `WithCache()` arms a copy an operator -/// makes. They build a fresh instance on every call, never a cached one - a snapshot manager a -/// write arms with a catalog loader is not one a scan should be served through afterwards. +/// The factories below take the options of the call, so an operator can serve a cache of its own +/// over the options the table was loaded with. Each call builds a fresh instance and never a +/// cached one: a snapshot manager a write armed with a catalog loader must not serve a scan. class FileStore { public: - /// @param file_system The file system the caller opened the table through, which wins over - /// the access configuration the schema's options may carry when it is not null. A null - /// one hands the resolution to that configuration, and to the default local file system - /// when the options name none. + /// Builds the store. It parses the schema's options and fails when they do not parse, so the + /// constructor is private and callers come here. + /// @param file_system The file system the caller opened the table through. It wins over the + /// access configuration in the schema's options. Null leaves the resolution to that + /// configuration, and to the default local file system when the options name none. /// @param root_path Root path of the table in the file system. - /// @param schema The schema the table was loaded with, whose partition keys name the partitions - /// the paths below are built for and whose options `GetOptions()` parses. Null when the + /// @param schema The schema the table was loaded with. Its partition keys name the partitions + /// the paths below are built for, and `GetOptions()` parses its options. Null when the /// caller had none, which the operators then report. /// @param branch The branch the store serves, normalized here. - /// @param schema_manager A schema manager to carry instead of building one - the one a context + /// @param schema_manager A schema manager to carry instead of building one: the one a context /// dispatch read the schema through, whose cache the operators then serve. - /// @param snapshot_manager A snapshot manager to carry instead of building one; a null one is - /// built over `root_path` on `branch`. + /// @param snapshot_manager A snapshot manager to carry instead of building one. Null builds one + /// over `root_path` on `branch`. /// @param fs_scheme_map The scheme-to-identifier map the caller's contexts resolve custom /// schemes through, kept so a copy resolves as this store does. - /// - /// Building the store parses the schema's options, so it fails when they do not parse - which - /// is why a store is built through this factory rather than the constructor. static Result> Create( const std::shared_ptr& file_system, const std::string& root_path, const std::shared_ptr& schema, const std::string& branch, - const std::shared_ptr& schema_manager = nullptr, - const std::shared_ptr& snapshot_manager = nullptr, - const std::map& fs_scheme_map = {}); - - /// Whether a copy's dynamic options change the file system its store resolves to. If they do, - /// the managers of the table it was copied from are not its to inherit: built over that - /// table's resolution, they would keep reading metadata through the backend the copy just - /// left. + const std::shared_ptr& schema_manager, + const std::shared_ptr& snapshot_manager, + const std::map& fs_scheme_map); + + /// Returns whether a copy's dynamic options change the file system its store resolves to. When + /// they do, the copy must not inherit the managers of the table it was copied from: they were + /// built over that table's resolution and would keep reading metadata through the backend the + /// copy left. static bool FileSystemResolutionMayChange( const TableSchema& schema, const std::map& dynamic_options); - /// Whether two options maps resolve to the same file system: every resolution key has to - /// agree, a missing one on both sides included. + /// Returns whether two options maps resolve to the same file system. Every resolution key has + /// to agree, including a key missing on both sides. static bool FileSystemResolutionAgrees(const std::map& left, const std::map& right); @@ -98,8 +94,8 @@ class FileStore { return branch_; } - /// The schema manager the operators read through, carrying the schema cache of `root_path` on - /// `CurrentBranch()`. + /// The schema manager the operators read through. It carries the schema cache of `root_path` + /// on `CurrentBranch()`. const std::shared_ptr& GetSchemaManager() const { return schema_manager_; } @@ -115,20 +111,20 @@ class FileStore { return schema_; } - /// The file system the table's data is accessed through: the one the options resolve, which - /// the file system the caller opened the table through wins over. + /// The file system the table's data is accessed through: the one the options resolve, unless + /// the caller opened the table through one of its own. const std::shared_ptr& GetFileSystem() const { return file_system_; } - /// The file system the caller opened the table through, as handed to `Create`: null when the + /// The file system the caller opened the table through, as handed to `Create`. Null when the /// call had none, so a copy resolves from its own options again. const std::shared_ptr& GetSpecifiedFileSystem() const { return specified_file_system_; } - /// The scheme-to-identifier map the options were resolved with, kept for a copy to resolve - /// through again. + /// The scheme-to-identifier map the options were resolved with, kept so a copy can resolve + /// through it again. const std::map& GetFsSchemeMap() const { return fs_scheme_map_; } @@ -137,59 +133,59 @@ class FileStore { return root_path_; } - /// The options the schema was loaded with on `CurrentBranch()`. They exist only over a schema: - /// an operator reads them after reporting that none was found when there is not. + /// The options of the schema loaded on `CurrentBranch()`. They exist only over a schema, so an + /// operator reports that no schema was found before it ever reads them. const CoreOptions& GetOptions() const { assert(core_options_.has_value()); return *core_options_; } - /// The path factory an operator builds its metadata paths through: over this store's root - /// path and the partition keys of its schema, which no schema change can alter, so an - /// operator serving a schema read after the table was loaded still names the same partitions. - /// The file format identifier is the caller's to name: a write names the format its level-0 - /// files are written in, which the options may set per level, while a scan, a read and a - /// commit name the table's format. + /// Builds the path factory an operator names its metadata paths through, over this store's + /// root path and the partition keys of its schema. No schema change can alter the partition + /// keys, so an operator serving a schema read after the table was loaded still names the same + /// partitions. The caller names the file format identifier: a write names the format of its + /// level-0 files, which the options may set per level, while a scan, a read and a commit name + /// the table's format. Result> NewPathFactory( const std::shared_ptr& arrow_schema, const CoreOptions& options, const std::string& file_format_identifier, const std::shared_ptr& memory_pool) const; - /// The snapshot manager a write or a commit reads snapshots through, over this store's root - /// path and branch. The file system is the caller's to name, from the options its operator - /// serves. A catalog that version-manages the table hands its snapshots out itself, so its - /// loader is attached and the catalog captured to keep it alive; such a catalog names the - /// table by an identifier, which the context must have supplied. + /// Builds the snapshot manager a write or a commit reads snapshots through, over this store's + /// root path and branch. The caller names the file system, from the options its operator + /// serves. A catalog that version-manages the table hands out its own snapshots, so its loader + /// is attached and the catalog is captured to keep it alive. Such a catalog names the table by + /// an identifier, which the context must have supplied. Result> NewSnapshotManager( const CoreOptions& options, const std::shared_ptr& catalog, const std::optional& catalog_identifier) const; - /// The manifest list a scan or a commit reads the table's manifest entries through. + /// Builds the manifest list a scan or a commit reads the table's manifest entries through. Result> NewManifestList( const CoreOptions& options, const std::shared_ptr& path_factory, const std::shared_ptr& memory_pool) const; - /// The manifest file a scan or a commit reads a single manifest's entries through, pruned by - /// `partition_schema`. + /// Builds the manifest file a scan or a commit reads a single manifest's entries through, + /// pruned by `partition_schema`. Result> NewManifestFile( const CoreOptions& options, const std::shared_ptr& path_factory, const std::shared_ptr& partition_schema, const std::shared_ptr& memory_pool) const; - /// The index manifest file a write, a scan or a commit reads the table's index manifest - /// through. + /// Builds the index manifest file a write, a scan or a commit reads the table's index + /// manifest through. Result> NewIndexManifestFile( const CoreOptions& options, const std::shared_ptr& path_factory, const std::shared_ptr& memory_pool) const; - /// The index file handler a write maintains its deletion vectors and indexes through, or a - /// scan reads them through. + /// Builds the index file handler a write maintains its deletion vectors and indexes through, + /// or a scan reads them through. Result> NewIndexFileHandler( const CoreOptions& options, const std::shared_ptr& path_factory, const std::shared_ptr& memory_pool) const; - /// The partition computer a commit names the partitions it writes with, over the partition - /// keys of this store's schema and the caller's arrow schema. + /// Builds the partition computer a commit names the partitions it writes with, over the + /// partition keys of this store's schema and the caller's arrow schema. Result> NewPartitionComputer( const std::shared_ptr& arrow_schema, const CoreOptions& options, const std::shared_ptr& memory_pool) const; diff --git a/src/paimon/core/table/file_store_table.cpp b/src/paimon/core/table/file_store_table.cpp index 7e68b3567..3945de5b7 100644 --- a/src/paimon/core/table/file_store_table.cpp +++ b/src/paimon/core/table/file_store_table.cpp @@ -214,7 +214,7 @@ Result> FileStoreTable::SwitchToBranch(const std::string& std::shared_ptr(std::move(branch_schema)), identifier_, CatalogUuid().value_or(""), target_branch, branch_schema_manager, /*snapshot_manager=*/nullptr, - store_->GetFsSchemeMap()); + store_->GetFsSchemeMap(), /*dynamic_options=*/{}); } Status FileStoreTable::RestoreRealtimeCommittedProgress( @@ -470,8 +470,8 @@ Result> FileStoreTable::NewRead( PAIMON_RETURN_NOT_OK(CheckOptionsAgreeWithTable(*loaded_schema, read_context->GetOptions())); // The read reads through the branch the table serves, not the context's own. PAIMON_ASSIGN_OR_RAISE(std::shared_ptr internal_context, - InternalReadContext::Create(read_context, loaded_schema, - store_->GetOptions(), &root_path_)); + InternalReadContext::CreateForTable(read_context, loaded_schema, + store_->GetOptions(), root_path_)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_read, CreateTableRead(internal_context, read_context->GetMemoryPool(), diff --git a/src/paimon/core/table/file_store_table.h b/src/paimon/core/table/file_store_table.h index 1fb307495..04b088ab8 100644 --- a/src/paimon/core/table/file_store_table.h +++ b/src/paimon/core/table/file_store_table.h @@ -43,44 +43,44 @@ class RealtimeContext; class Schema; class SplitGenerator; -/// The managed table a catalog hands out: one with snapshots and manifests, which the schema under -/// its root path describes. A format table - a directory of plain data files - is a `FormatTable`, -/// the other kind of `Table`. +/// The managed table a catalog hands out: one with snapshots and manifests, described by the +/// schema under its root path. A format table - a directory of plain data files - is a +/// `FormatTable`, the other kind of `Table`. /// -/// This is the flow the four operators share, Java Paimon's `AbstractFileStoreTable`: what the -/// table was constructed with - the schema, file system, root path, branch and managers - decides -/// what every operator serves, so it is read here once and handed down. The schema's primary keys -/// decide the kind of managed table it is - `AppendOnlyFileStoreTable` or -/// `PrimaryKeyFileStoreTable`, the `Create` factory below - and that kind builds the pieces the -/// flows hand down: the write itself, the scans, the read and the split generator. +/// The four operators share one flow, the one Java Paimon's `AbstractFileStoreTable` carries. What +/// the table was loaded with - schema, file system, root path, branch and managers - decides what +/// every operator serves, so it is read once here and handed down. The schema's primary keys decide +/// the kind of managed table it is, `AppendOnlyFileStoreTable` or `PrimaryKeyFileStoreTable`, which +/// the `Create` factory below picks. That kind then builds the pieces the flow hands down: the +/// write itself, the scans, the read and the split generator. class FileStoreTable : public Table { public: - /// The managed table the schema describes, built as the kind its primary keys name: an - /// `AppendOnlyFileStoreTable` when there are none, a `PrimaryKeyFileStoreTable` when there - /// are, the way Java Paimon's `FileStoreTableFactory` does. Primary keys cannot be added or - /// dropped by a schema change, so the kind is the same for every schema the root path held. - /// @param schema The schema the table was loaded with, whose primary keys name the kind and - /// whose options the operators serve. Null when the caller had none, in which case the - /// table is built as the append-only kind and its operators report that none was found. + /// Builds the managed table the schema describes, as the kind its primary keys name: an + /// `AppendOnlyFileStoreTable` when there are none, a `PrimaryKeyFileStoreTable` when there are. + /// Java Paimon's `FileStoreTableFactory` picks the kind the same way. A schema change can + /// neither add nor drop primary keys, so every schema the root path ever held names one kind. + /// @param schema The schema the table was loaded with. Its primary keys name the kind and its + /// options are what the operators serve. Null when the caller had none, in which case + /// the table is built as the append-only kind and its operators report that none was + /// found. /// @param branch The branch the table serves, which the managers below read and write on. - /// @param schema_manager A schema manager to carry instead of building one - see + /// @param schema_manager A schema manager to carry instead of building one. See /// `FileStore::Create`. - /// @param snapshot_manager A snapshot manager to carry instead of building one, the way `Copy` - /// hands its table's over so both share one - see `FileStore::Create`. + /// @param snapshot_manager A snapshot manager to carry instead of building one. `Copy` hands + /// its table's over so both share one. See `FileStore::Create`. /// @param fs_scheme_map The scheme-to-identifier map the caller's contexts resolve custom /// schemes through, which the store's options resolve through as well. `Copy` hands its /// table's map over. /// @param dynamic_options The options a copy was given over the ones its schema stored, which - /// `Copy` accumulates - see the member of the same name. + /// `Copy` accumulates. See the member of the same name. static Result> Create( const std::shared_ptr& file_system, const std::string& root_path, const std::shared_ptr& schema, const Identifier& identifier, - const std::string& uuid = "", - const std::string& branch = BranchManager::DEFAULT_MAIN_BRANCH, - const std::shared_ptr& schema_manager = nullptr, - const std::shared_ptr& snapshot_manager = nullptr, - const std::map& fs_scheme_map = {}, - const std::map& dynamic_options = {}); + const std::string& uuid, const std::string& branch, + const std::shared_ptr& schema_manager, + const std::shared_ptr& snapshot_manager, + const std::map& fs_scheme_map, + const std::map& dynamic_options); const std::string& CurrentBranch() const { return store_->CurrentBranch(); @@ -102,15 +102,15 @@ class FileStoreTable : public Table { return identifier_.GetFullName(); } - /// The file system, root path and identifier stay as they are: they name this table, which an - /// option does not rename. + /// Returns a copy that serves `dynamic_options`. The file system, root path and identifier + /// stay as they are: they name this table, and an option does not rename it. Result> Copy( const std::map& dynamic_options) const override; /// Returns the table of `branch`, as Java Paimon's `switchToBranch` does. The branch switched /// to is served as it is: its schema is read from its own directory, so this table's dynamic - /// options are not carried over. Switching to the branch already served hands back a copy of - /// this table, schema and managers shared, and a branch with no schema does not exist. + /// options are not carried over. Switching to the branch this table already serves hands back a + /// copy of it with the schema and the managers shared. A branch with no schema does not exist. Result> SwitchToBranch(const std::string& branch) const; Result> NewScan(std::unique_ptr context) const override; @@ -124,47 +124,47 @@ class FileStoreTable : public Table { std::unique_ptr context) const override; private: - /// The options a context carries against the ones the table was loaded with. A table serves - /// the options it was loaded with, so an option the context contradicts is refused rather than - /// honoured - a caller that means to change them copies the table. A table a load hands back - /// carries the context's own options merged in - `FileStoreTableFactory` does the merging - so - /// it agrees by construction; the refusal belongs to a table the caller held already, whose - /// load is over. A key the table was not loaded with is no contradiction: it names a runtime - /// setting - a scan mode, a batch size - which is none of the table's business, and is served - /// as given. + /// Checks the options a context carries against the ones the table was loaded with. A table + /// serves the options it was loaded with, so an option the context contradicts is refused: a + /// caller that means to change them copies the table. A table a load hands back carries the + /// context's own options merged in - `FileStoreTableFactory` merges them - so it agrees by + /// construction. The refusal belongs to a table the caller already held, whose load is over. A + /// key the table was not loaded with is no contradiction: it names a runtime setting, a scan + /// mode or a batch size, which is none of the table's business and is served as given. static Status CheckOptionsAgreeWithTable( const TableSchema& table_schema, const std::map& context_options); - /// The read `NewRead` hands back, from the internal context it read the schema into: the path - /// factory comes from the store, and the reader is selected by the schema the read serves - the - /// primary keys it carries - rather than by the table's own kind. + /// Builds the read `NewRead` hands back, from the internal context it read the schema into. The + /// path factory comes from the store. The reader is picked by the schema the read serves - the + /// primary keys it carries - not by the table's own kind. Result> CreateTableRead( const std::shared_ptr& internal_context, const std::shared_ptr& memory_pool, const std::shared_ptr& executor) const; protected: - /// The base constructor the kinds of managed table build through: the store carries every state - /// the kinds hand down - see `Create` for what each piece of it is. + /// The base constructor the kinds of managed table build through. The store carries every state + /// they hand down; see `Create` for what each piece of it is. FileStoreTable(const std::shared_ptr& store, const Identifier& identifier, const std::string& uuid); - /// The committed progress a real-time write restores, from the latest snapshot the snapshot - /// manager reads: where the write picks up after, which both kinds ask for the same way. + /// Restores the committed progress of a real-time write from the latest snapshot the snapshot + /// manager reads. It tells the write where to pick up, and both kinds ask for it the same way. static Status RestoreRealtimeCommittedProgress( const std::shared_ptr& realtime_context, const std::shared_ptr& snapshot_manager, const CoreOptions& options); - /// The scan the table's root path names, which the kinds of file store scan read from. + /// Returns the scan restricted to the table's root path, which the kinds of file store scan + /// read from. static std::unique_ptr WithTablePath(std::unique_ptr&& scan, const std::string& root_path); - /// The write of this kind of table, which `NewWrite` hands the shared flow's pieces to. The - /// schema and the branch the write serves are the table's own. + /// Builds the write of this kind of table. `NewWrite` hands the pieces of the shared flow to + /// it. The schema and the branch the write serves are the table's own. /// @param io_manager The spill manager the context's temp directory names, or null when the /// context named none. - /// @param ignore_previous_files Whether the write starts over rather than appending, which - /// the flow resolved with this kind's own rules. + /// @param ignore_previous_files Whether the write starts over rather than appends, which the + /// flow resolved with this kind's own rules. virtual Result> CreateWrite( const WriteContext& ctx, const std::shared_ptr& arrow_schema, const std::shared_ptr& partition_schema, const CoreOptions& options, @@ -172,8 +172,8 @@ class FileStoreTable : public Table { const std::shared_ptr& snapshot_manager, const std::shared_ptr& io_manager, bool ignore_previous_files) const = 0; - /// The scan of this kind of table, which `NewScan` hands the manifest list and file it built - /// to, over the table's own schema and managers. + /// Builds the scan of this kind of table. `NewScan` hands it the manifest list and the manifest + /// file it built, over the table's own schema and managers. virtual Result> NewFileStoreScan( const std::shared_ptr& manifest_list, const std::shared_ptr& manifest_file, @@ -181,12 +181,12 @@ class FileStoreTable : public Table { const std::shared_ptr& executor, const std::shared_ptr& memory_pool, const ScanContext* context) const = 0; - /// The split generator of this kind of table, Java Paimon's `splitGenerator()`. + /// Builds the split generator of this kind of table, Java Paimon's `splitGenerator()`. virtual Result> NewSplitGenerator( const CoreOptions& core_options) const = 0; - /// The manifest entry scan supplier of this kind of table, which `NewCommit` hands the manifest - /// list and file it built to. + /// Builds the manifest entry scan supplier of this kind of table. `NewCommit` hands it the + /// manifest list and the manifest file it built. virtual CommitScanner::ScanSupplier NewCommitScanSupplier( const std::shared_ptr& snapshot_manager, const std::shared_ptr& manifest_list, @@ -195,13 +195,13 @@ class FileStoreTable : public Table { const std::shared_ptr& executor, const std::shared_ptr& pool) const = 0; - /// The storage this table serves - see `FileStore`. + /// The storage this table reads and writes through. See `FileStore`. std::shared_ptr store_; - /// The dynamic options the table was loaded with over the ones its schema stored, which `Copy` - /// accumulates into the copy it derives. They stay separate from the schema they were merged - /// into because the fallback reader of a read needs them alone: it serves another branch's - /// schema, over which the same options are merged again. + /// The dynamic options the table was loaded with, over the ones its schema stored. `Copy` + /// accumulates them into the copy it derives. They stay separate from the schema they were + /// merged into because a fallback reader needs them on their own: it serves another branch's + /// schema and merges the same options over it again. std::map dynamic_options_; }; diff --git a/src/paimon/core/table/file_store_table_factory.cpp b/src/paimon/core/table/file_store_table_factory.cpp index 2b5c9a40c..d5082ef73 100644 --- a/src/paimon/core/table/file_store_table_factory.cpp +++ b/src/paimon/core/table/file_store_table_factory.cpp @@ -125,7 +125,7 @@ Result> TableOfLoadedSchema( const std::string& branch, const std::map& options, const std::shared_ptr& schema_manager, const std::shared_ptr& latest_schema, const std::optional& identifier, - const std::string& uuid = "", const std::map& fs_scheme_map = {}) { + const std::string& uuid, const std::map& fs_scheme_map) { PAIMON_ASSIGN_OR_RAISE(std::shared_ptr format_table, TryLoadFormatTable(file_system, path, branch, options, latest_schema)); if (format_table != nullptr) { @@ -146,10 +146,11 @@ Result> TableOfLoadedSchema( !FileStore::FileSystemResolutionAgrees(latest_schema->Options(), options))) { manager = nullptr; } - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table, - FileStoreTable::Create(explicit_file_system, path, latest_schema, - table_identifier, uuid, branch, manager, - /*snapshot_manager=*/nullptr, fs_scheme_map)); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr table, + FileStoreTable::Create(explicit_file_system, path, latest_schema, table_identifier, uuid, + branch, manager, /*snapshot_manager=*/nullptr, fs_scheme_map, + /*dynamic_options=*/{})); return WithContextOptions(table, latest_schema, options); } @@ -167,7 +168,7 @@ Result> ManagedTableOf( const std::shared_ptr& explicit_file_system, const std::string& path, const std::string& branch, const std::map& options, const std::optional& schema_json, const std::optional& identifier, - const std::string& uuid = "", const std::map& fs_scheme_map = {}) { + const std::string& uuid, const std::map& fs_scheme_map) { auto schema_manager = std::make_shared(file_system, path, branch); std::shared_ptr latest_schema; if (branch == BranchManager::DEFAULT_MAIN_BRANCH && schema_json) { @@ -230,7 +231,8 @@ Result> FileStoreTableFactory::Create(const ScanContext& const std::string branch = BranchManager::NormalizeBranch(tmp_options.GetBranch()); return ManagedTableOf(tmp_options.GetFileSystem(), ctx.GetSpecificFileSystem(), ctx.GetPath(), branch, ctx.GetOptions(), ctx.GetSpecificTableSchema(), - ctx.GetIdentifier()); + ctx.GetIdentifier(), + /*uuid=*/"", /*fs_scheme_map=*/{}); } /// The table a read context names: the format table the builder was handed, a system table behind @@ -340,7 +342,7 @@ Result> FileStoreTableFactory::Create(const CommitContext // not dispatch on different schemas. return ManagedTableOf(tmp_options.GetFileSystem(), specific_fs, root_path, branch, ctx.GetOptions(), catalog_table_schema, ctx.GetIdentifier(), - ctx.GetTableId().value_or("")); + ctx.GetTableId().value_or(""), /*fs_scheme_map=*/{}); } } // namespace paimon diff --git a/src/paimon/core/table/file_store_table_factory.h b/src/paimon/core/table/file_store_table_factory.h index 175834ff6..a1bb77062 100644 --- a/src/paimon/core/table/file_store_table_factory.h +++ b/src/paimon/core/table/file_store_table_factory.h @@ -30,10 +30,10 @@ class ScanContext; class Table; class WriteContext; -/// The table a context names, Java Paimon's `FileStoreTableFactory`: the format table the builder -/// was handed, a system table behind a `T$system` path, or the table the schema under the -/// context's path says is there. The four operators' static `Create` entries all dispatch on the -/// same question, so one place answers it and they cannot disagree. +/// Loads the table a context names, as Java Paimon's `FileStoreTableFactory` does: the format +/// table the builder was handed, the system table behind a `T$system` path, or the managed table +/// the schema under the context's path describes. The static `Create` of all four operators asks +/// the same question, so one place answers it and they cannot disagree. class FileStoreTableFactory { public: FileStoreTableFactory() = delete; diff --git a/src/paimon/core/table/format/format_table.cpp b/src/paimon/core/table/format/format_table.cpp index d4b611eac..f14ac66d1 100644 --- a/src/paimon/core/table/format/format_table.cpp +++ b/src/paimon/core/table/format/format_table.cpp @@ -224,6 +224,9 @@ Result> FormatTable::GetArrowSchema() const { return schema_->GetArrowSchema(); } +// The refusals below use two codes. `Invalid` is a setting that clashes with what a format table +// is, and will never work: no snapshots, no buckets, no real-time store, no manifests, no +// catalog-side version. `NotImplemented` is a capability this code lacks today and could gain. Result> FormatTable::NewScan( std::unique_ptr context) const { std::shared_ptr shared_context = std::move(context); @@ -233,15 +236,15 @@ Result> FormatTable::NewScan( // Anything a `ScanContext` carries that a format table cannot honour is refused rather than // silently dropped. if (shared_context->IsStreamingMode()) { - return Status::NotImplemented( + return Status::Invalid( "a format table has no snapshots, so there is nothing for a streaming scan to follow"); } if (shared_context->GetRealtimeContext() != nullptr) { - return Status::NotImplemented( + return Status::Invalid( "a format table has no real-time store to union with what is on disk"); } if (shared_context->GetGlobalIndexResult() != nullptr) { - return Status::NotImplemented("a format table carries no global index"); + return Status::Invalid("a format table carries no global index"); } std::map partition_filter; std::shared_ptr filters = shared_context->GetScanFilters(); @@ -253,12 +256,12 @@ Result> FormatTable::NewScan( "implemented yet; give the partition values as a partition filter instead"); } if (filters->GetBucketFilter()) { - return Status::NotImplemented("a format table has no buckets to filter by"); + return Status::Invalid("a format table has no buckets to filter by"); } const std::vector>& partition_filters = filters->GetPartitionFilters(); if (partition_filters.size() > 1) { - return Status::NotImplemented("a format table scan takes at most one partition filter"); + return Status::Invalid("a format table scan takes at most one partition filter"); } if (partition_filters.size() == 1) { partition_filter = partition_filters.front(); @@ -298,12 +301,12 @@ Result> FormatTable::NewWrite( "format table"); } if (context->IsStreamingMode()) { - return Status::NotImplemented( + return Status::Invalid( "a format table has no snapshots, so there is nothing a streaming write could " "commit against"); } if (context->GetRealtimeContext() != nullptr) { - return Status::NotImplemented("a format table has no real-time store to write into"); + return Status::Invalid("a format table has no real-time store to write into"); } if (!context->GetWriteSchema().empty()) { return Status::NotImplemented( @@ -313,8 +316,7 @@ Result> FormatTable::NewWrite( // A write id prefixes a postpone-bucket writer's files so that one compaction reader can // put them back in order. A format table has no buckets, so it would identify nothing. if (context->GetWriteId().has_value()) { - return Status::NotImplemented( - "a format table has no buckets, so a write id would name nothing"); + return Status::Invalid("a format table has no buckets, so a write id would name nothing"); } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr format_write, FormatTableFileStoreWrite::Create(table, context->GetMemoryPool())); @@ -327,26 +329,26 @@ Result> FormatTable::NewCommit( PAIMON_ASSIGN_OR_RAISE(std::shared_ptr given_table, CopySelf(context->GetOptions())); if (!context->IgnoreEmptyCommit()) { - return Status::NotImplemented( + return Status::Invalid( "a format table cannot record an empty commit: keeping one means writing a " "snapshot that adds no files, and there are no snapshots here"); } if (context->UseRESTCatalogCommit()) { - return Status::NotImplemented( + return Status::Invalid( "a format table commits by renaming files into place, not by sending a snapshot " "to a rest catalog"); } if (context->GetTableId()) { - return Status::NotImplemented( + return Status::Invalid( "a format table sends no commit table request, so there is nowhere to put a table id"); } if (context->GetCatalog() != nullptr) { - return Status::NotImplemented( + return Status::Invalid( "a format table keeps no snapshot for a catalog to take, so it commits by renaming " "files into place rather than through the catalog it was loaded from"); } if (context->AppendCommitCheckConflict()) { - return Status::NotImplemented( + return Status::Invalid( "a format table has no manifests to check a concurrent commit against"); } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr format_commit, diff --git a/src/paimon/core/table/format/format_table_test.cpp b/src/paimon/core/table/format/format_table_test.cpp index 2dac6c102..84fbce351 100644 --- a/src/paimon/core/table/format/format_table_test.cpp +++ b/src/paimon/core/table/format/format_table_test.cpp @@ -2766,6 +2766,35 @@ TEST(FormatTableTest, TestTheGenericEntryPointsReachAFormatTable) { reader->Close(); } +TEST(FormatTableTest, TestACatalogInTheContextIsRefusedAlikeByWriteAndCommit) { + // `WithCatalog()` asks for a catalog-side version of the commit, which a format table has no + // way to produce. Both halves of the write path answer with the same code, so a caller reads + // one kind of failure out of them rather than "cannot be done" from one and "not yet" from + // the other. + std::unique_ptr dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + ASSERT_OK_AND_ASSIGN(std::shared_ptr table, + CreateTable(dir->GetFileSystem(), dir->Str(), {"dt"})); + + auto catalog = std::make_shared(); + catalog->SetTableSchema(table->LatestSchema()); + Identifier identifier("db", "fmt"); + + WriteContextBuilder write_builder(dir->Str(), "test-user"); + write_builder.WithCatalog(catalog, identifier); + ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, write_builder.Finish()); + Result> write = table->NewWrite(std::move(write_context)); + ASSERT_FALSE(write.ok()); + ASSERT_TRUE(write.status().IsInvalid()); + + CommitContextBuilder commit_builder(dir->Str(), "test-user"); + commit_builder.WithCatalog(catalog, identifier); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, commit_builder.Finish()); + Result> commit = table->NewCommit(std::move(commit_context)); + ASSERT_FALSE(commit.ok()); + ASSERT_TRUE(commit.status().IsInvalid()); +} + TEST(FormatTableTest, TestTheGenericEntryPointsTakeALoadedFormatTable) { // A table a catalog keeps the schema for, the way a rest catalog does. Nothing under its // location says that it is a format table, nor that everything below the location is data, so diff --git a/src/paimon/core/table/primary_key_file_store_table.h b/src/paimon/core/table/primary_key_file_store_table.h index ce7547a71..2d3390e86 100644 --- a/src/paimon/core/table/primary_key_file_store_table.h +++ b/src/paimon/core/table/primary_key_file_store_table.h @@ -43,7 +43,7 @@ class PrimaryKeyFileStoreTable : public FileStoreTable { /// The constructor `FileStoreTable::Create` builds a primary key table through: the store /// carries every state, and the base class holds it. PrimaryKeyFileStoreTable(const std::shared_ptr& store, const Identifier& identifier, - const std::string& uuid = ""); + const std::string& uuid); private: Result> CreateWrite( diff --git a/src/paimon/core/table/source/table_read_test.cpp b/src/paimon/core/table/source/table_read_test.cpp index 3cd3b22de..3e79c537d 100644 --- a/src/paimon/core/table/source/table_read_test.cpp +++ b/src/paimon/core/table/source/table_read_test.cpp @@ -137,7 +137,10 @@ TEST(TableReadTest, TestFullTextMatchExecutionIsNotImplemented) { ASSERT_OK_AND_ASSIGN(std::shared_ptr schema, schema_manager.ReadSchema(0)); ASSERT_OK_AND_ASSIGN( std::shared_ptr table, - FileStoreTable::Create(fs, table_path, schema, Identifier("append_09.db", "append_09"))); + FileStoreTable::Create(fs, table_path, schema, Identifier("append_09.db", "append_09"), + /*uuid=*/"", /*branch=*/BranchManager::DEFAULT_MAIN_BRANCH, + /*schema_manager=*/nullptr, /*snapshot_manager=*/nullptr, + /*fs_scheme_map=*/{}, /*dynamic_options=*/{})); for (bool table_entry : {false, true}) { ReadContextBuilder builder(table_path); builder.SetReadFieldNames({"f0", "f1"}).SetPredicate(predicate); @@ -164,7 +167,10 @@ TEST(TableReadTest, TestPerFieldScoreExecutionIsNotImplemented) { ASSERT_OK_AND_ASSIGN(std::shared_ptr schema, schema_manager.ReadSchema(0)); ASSERT_OK_AND_ASSIGN( std::shared_ptr table, - FileStoreTable::Create(fs, path, schema, Identifier("append_09.db", "append_09"))); + FileStoreTable::Create(fs, path, schema, Identifier("append_09.db", "append_09"), + /*uuid=*/"", /*branch=*/BranchManager::DEFAULT_MAIN_BRANCH, + /*schema_manager=*/nullptr, /*snapshot_manager=*/nullptr, + /*fs_scheme_map=*/{}, /*dynamic_options=*/{})); for (bool table_entry : {false, true}) { ReadContextBuilder builder(path); builder.SetFullTextSearch(search).SetReadFieldNames({"f1", "_f0_INDEX_SCORE"}); diff --git a/src/paimon/core/table/system/audit_log_system_table.cpp b/src/paimon/core/table/system/audit_log_system_table.cpp index 09fd6328f..0bf82ead3 100644 --- a/src/paimon/core/table/system/audit_log_system_table.cpp +++ b/src/paimon/core/table/system/audit_log_system_table.cpp @@ -359,16 +359,19 @@ std::unique_ptr CreateChangelogBatchReader( AuditLogSystemTable::AuditLogSystemTable(std::shared_ptr fs, std::string table_path, std::shared_ptr table_schema, std::map options) - : SystemTable(fs, table_path), + : AuditLogSystemTable(std::move(fs), table_path, std::move(table_schema), std::move(options), + ComposeName(table_path, kName)) {} + +AuditLogSystemTable::AuditLogSystemTable(std::shared_ptr fs, std::string table_path, + std::shared_ptr table_schema, + std::map options, + std::string name) + : SystemTable(fs, table_path, std::move(name)), fs_(std::move(fs)), table_path_(std::move(table_path)), table_schema_(std::move(table_schema)), options_(std::move(options)) {} -std::string AuditLogSystemTable::Name() const { - return kName; -} - Result> AuditLogSystemTable::ArrowSchema() const { std::shared_ptr row_kind_field = DataField::ConvertDataFieldToArrowField(SpecialFields::RowKind()); @@ -398,6 +401,8 @@ Result> AuditLogSystemTable::NewScan( .WithMemoryPool(context->GetMemoryPool()) .WithExecutor(context->GetExecutor()) // The view's own file system when the context names none, rather than the default backend. + // A `ScanContext` carries no scheme-to-identifier map, so unlike `CreateDataReadContext` + // below there is no map here to resolve through instead. .WithFileSystem(context->GetSpecificFileSystem() != nullptr ? context->GetSpecificFileSystem() : GetFileSystem()) diff --git a/src/paimon/core/table/system/audit_log_system_table.h b/src/paimon/core/table/system/audit_log_system_table.h index 7395c9797..4cc324b1d 100644 --- a/src/paimon/core/table/system/audit_log_system_table.h +++ b/src/paimon/core/table/system/audit_log_system_table.h @@ -58,12 +58,17 @@ class AuditLogSystemTable : public SystemTable { std::shared_ptr table_schema, std::map options); - std::string Name() const override; Result> ArrowSchema() const override; Result> NewScan(std::unique_ptr context) const override; Result> NewRead(std::unique_ptr context) const override; protected: + /// For a derived changelog view that answers with a kind of its own: `BinlogSystemTable` names + /// itself `T$binlog`, not the `T$audit_log` the public constructor composes. + AuditLogSystemTable(std::shared_ptr fs, std::string table_path, + std::shared_ptr table_schema, + std::map options, std::string name); + Result> NewChangelogRead( const std::shared_ptr& context, std::shared_ptr converter) const; diff --git a/src/paimon/core/table/system/binlog_system_table.cpp b/src/paimon/core/table/system/binlog_system_table.cpp index a3c39078d..8299f17f5 100644 --- a/src/paimon/core/table/system/binlog_system_table.cpp +++ b/src/paimon/core/table/system/binlog_system_table.cpp @@ -74,12 +74,8 @@ std::shared_ptr CreateBinlogBatchConverter() { BinlogSystemTable::BinlogSystemTable(std::shared_ptr fs, std::string table_path, std::shared_ptr table_schema, std::map options) - : AuditLogSystemTable(std::move(fs), std::move(table_path), std::move(table_schema), - std::move(options)) {} - -std::string BinlogSystemTable::Name() const { - return kName; -} + : AuditLogSystemTable(std::move(fs), table_path, std::move(table_schema), std::move(options), + ComposeName(table_path, kName)) {} Result> BinlogSystemTable::ArrowSchema() const { std::shared_ptr rowkind_field = diff --git a/src/paimon/core/table/system/binlog_system_table.h b/src/paimon/core/table/system/binlog_system_table.h index 3a939fcef..3a28ffa93 100644 --- a/src/paimon/core/table/system/binlog_system_table.h +++ b/src/paimon/core/table/system/binlog_system_table.h @@ -40,7 +40,6 @@ class BinlogSystemTable : public AuditLogSystemTable { std::shared_ptr table_schema, std::map options); - std::string Name() const override; Result> ArrowSchema() const override; Result> NewRead(std::unique_ptr context) const override; }; diff --git a/src/paimon/core/table/system/global_system_tables.cpp b/src/paimon/core/table/system/global_system_tables.cpp index a66362b39..92f10c49d 100644 --- a/src/paimon/core/table/system/global_system_tables.cpp +++ b/src/paimon/core/table/system/global_system_tables.cpp @@ -126,6 +126,18 @@ Result IsEnabled(const GlobalSystemTableRegistryEntry& entry, CatalogOptionsSystemTable::kEnabledOption, false); } +// Whether a table reads its rows by enumerating what the catalog holds, and so cannot be built +// without one. +bool RequiresCatalog(const std::string& table_name) { + std::string normalized = StringUtils::ToLowerCase(table_name); + for (const auto& entry : GlobalSystemTableRegistry()) { + if (entry.name == normalized) { + return entry.requires_catalog; + } + } + return false; +} + struct CatalogTableInfo { std::string database_name; std::string table_name; @@ -134,11 +146,16 @@ struct CatalogTableInfo { // Match Java CatalogUtils::listAllTables: tolerate databases or tables removed concurrently, but // propagate all other catalog and schema errors instead of returning incomplete system-table rows. -Result> LoadAllDataTables(const Catalog& catalog) { +// `table_name` names the table asking, for the refusal a missing catalog reports. +Result> LoadAllDataTables( + const std::shared_ptr& catalog, const std::string& table_name) { + if (catalog == nullptr) { + return Status::NotImplemented("global system table requires catalog context: ", table_name); + } std::vector result; - PAIMON_ASSIGN_OR_RAISE(std::vector databases, catalog.ListDatabases()); + PAIMON_ASSIGN_OR_RAISE(std::vector databases, catalog->ListDatabases()); for (const std::string& database : databases) { - Result> tables_result = catalog.ListTables(database); + Result> tables_result = catalog->ListTables(database); if (!tables_result.ok()) { if (tables_result.status().IsNotExist()) { continue; @@ -147,7 +164,7 @@ Result> LoadAllDataTables(const Catalog& catalog) } for (const std::string& table : tables_result.value()) { Identifier identifier(database, table); - Result> schema_result = catalog.LoadTableSchema(identifier); + Result> schema_result = catalog->LoadTableSchema(identifier); if (!schema_result.ok()) { if (schema_result.status().IsNotExist()) { continue; @@ -288,6 +305,16 @@ Result GlobalSystemTableLoader::IsSupported( } Result> GlobalSystemTableLoader::Load( + const std::string& table_name, const GlobalSystemTableContext& context) { + // Refused here rather than at the first read: a table handed back to a caller that can never + // build its rows is a failure the caller finds out about late and far from its cause. + if (context.catalog == nullptr && RequiresCatalog(table_name)) { + return Status::NotImplemented("global system table requires catalog context: ", table_name); + } + return LoadForSchema(table_name, context); +} + +Result> GlobalSystemTableLoader::LoadForSchema( const std::string& table_name, const GlobalSystemTableContext& context) { std::string normalized = StringUtils::ToLowerCase(table_name); for (const auto& entry : GlobalSystemTableRegistry()) { @@ -296,10 +323,6 @@ Result> GlobalSystemTableLoader::Load( if (!enabled) { return Status::NotExist("global system table is disabled: ", table_name); } - if (entry.requires_catalog && context.catalog == nullptr) { - return Status::NotImplemented("global system table requires catalog context: ", - table_name); - } PAIMON_ASSIGN_OR_RAISE(std::shared_ptr system_table, entry.factory(context)); // The table is handed out through the `Table` interface, which serves the schema @@ -330,11 +353,7 @@ Result> GlobalSystemTableLoader::GetSupportedTableNames // ============================================================================= CatalogOptionsSystemTable::CatalogOptionsSystemTable(GlobalSystemTableContext context) - : InMemorySystemTable(context.fs, "sys/catalog_options"), context_(std::move(context)) {} - -std::string CatalogOptionsSystemTable::Name() const { - return kName; -} + : InMemorySystemTable(context.fs, "sys/catalog_options", kName), context_(std::move(context)) {} Result> CatalogOptionsSystemTable::ArrowSchema() const { return arrow::schema({ @@ -366,11 +385,8 @@ Result> CatalogOptionsSystemTable::BuildRows() const { // ============================================================================= AllTableOptionsSystemTable::AllTableOptionsSystemTable(GlobalSystemTableContext context) - : InMemorySystemTable(context.fs, "sys/all_table_options"), context_(std::move(context)) {} - -std::string AllTableOptionsSystemTable::Name() const { - return kName; -} + : InMemorySystemTable(context.fs, "sys/all_table_options", kName), + context_(std::move(context)) {} Result> AllTableOptionsSystemTable::ArrowSchema() const { return arrow::schema({ @@ -386,7 +402,7 @@ Result> AllTableOptionsSystemTable::BuildRows() const { std::vector rows; PAIMON_ASSIGN_OR_RAISE(std::vector tables, - LoadAllDataTables(*context_.catalog)); + LoadAllDataTables(context_.catalog, kName)); for (const CatalogTableInfo& table : tables) { for (const auto& [key, value] : table.schema->Options()) { GenericRow row(schema->num_fields()); @@ -405,11 +421,7 @@ Result> AllTableOptionsSystemTable::BuildRows() const { // ============================================================================= TablesSystemTable::TablesSystemTable(GlobalSystemTableContext context) - : InMemorySystemTable(context.fs, "sys/tables"), context_(std::move(context)) {} - -std::string TablesSystemTable::Name() const { - return kName; -} + : InMemorySystemTable(context.fs, "sys/tables", kName), context_(std::move(context)) {} Result> TablesSystemTable::ArrowSchema() const { return arrow::schema({ @@ -435,7 +447,7 @@ Result> TablesSystemTable::BuildRows() const { std::vector rows; PAIMON_ASSIGN_OR_RAISE(std::vector tables, - LoadAllDataTables(*context_.catalog)); + LoadAllDataTables(context_.catalog, kName)); for (const CatalogTableInfo& table : tables) { const std::shared_ptr& data_schema = table.schema; @@ -477,11 +489,7 @@ Result> TablesSystemTable::BuildRows() const { // ============================================================================= PartitionsSystemTable::PartitionsSystemTable(GlobalSystemTableContext context) - : InMemorySystemTable(context.fs, "sys/partitions"), context_(std::move(context)) {} - -std::string PartitionsSystemTable::Name() const { - return kName; -} + : InMemorySystemTable(context.fs, "sys/partitions", kName), context_(std::move(context)) {} Result> PartitionsSystemTable::ArrowSchema() const { return arrow::schema({ @@ -501,7 +509,7 @@ Result> PartitionsSystemTable::BuildRows() const { std::vector rows; PAIMON_ASSIGN_OR_RAISE(std::vector tables, - LoadAllDataTables(*context_.catalog)); + LoadAllDataTables(context_.catalog, kName)); for (const CatalogTableInfo& table : tables) { const std::shared_ptr& data_schema = table.schema; Identifier id(table.database_name, table.table_name); diff --git a/src/paimon/core/table/system/global_system_tables.h b/src/paimon/core/table/system/global_system_tables.h index eadc83af1..280f57479 100644 --- a/src/paimon/core/table/system/global_system_tables.h +++ b/src/paimon/core/table/system/global_system_tables.h @@ -33,14 +33,11 @@ class FileSystem; /// Context passed to global system table constructors, providing catalog-level /// access for enumerating databases, tables, and reading metadata. struct GlobalSystemTableContext { - /// The catalog the table reads its rows through, non-owning: the rows are built lazily, - /// so a table that outlives the call that built it must be handed the shared ownership - /// below too, or this pointer dangles at the first `BuildRows`. - const Catalog* catalog = nullptr; - /// The shared ownership a returned table keeps of its catalog, so its lazy `BuildRows` - /// cannot read through one already destroyed. Empty where the table is transient - used - /// within the call, before the catalog can go away. - std::shared_ptr catalog_retainer; + /// The catalog this table reads its rows through. Rows are built lazily, so a table handed back + /// to a caller keeps shared ownership of it. Empty when the table is transient and only its + /// schema is read. `GlobalSystemTableLoader::Load` refuses a table that enumerates data tables + /// when this is empty. + std::shared_ptr catalog; std::shared_ptr fs; std::string warehouse; std::map catalog_options; @@ -55,7 +52,6 @@ class CatalogOptionsSystemTable : public InMemorySystemTable { explicit CatalogOptionsSystemTable(GlobalSystemTableContext context); - std::string Name() const override; Result> ArrowSchema() const override; Result> BuildRows() const override; @@ -71,7 +67,6 @@ class AllTableOptionsSystemTable : public InMemorySystemTable { explicit AllTableOptionsSystemTable(GlobalSystemTableContext context); - std::string Name() const override; Result> ArrowSchema() const override; Result> BuildRows() const override; @@ -87,7 +82,6 @@ class TablesSystemTable : public InMemorySystemTable { explicit TablesSystemTable(GlobalSystemTableContext context); - std::string Name() const override; Result> ArrowSchema() const override; Result> BuildRows() const override; @@ -103,7 +97,6 @@ class PartitionsSystemTable : public InMemorySystemTable { explicit PartitionsSystemTable(GlobalSystemTableContext context); - std::string Name() const override; Result> ArrowSchema() const override; Result> BuildRows() const override; @@ -120,9 +113,17 @@ class GlobalSystemTableLoader { static Result IsSupported(const std::string& table_name, const std::map& catalog_options); + /// Loads a global system table for a caller to read. A table that enumerates data tables needs + /// the catalog in the context, so it is refused here when the context carries none. static Result> Load(const std::string& table_name, const GlobalSystemTableContext& context); + /// Loads a global system table for the schema it exposes alone, the way + /// `Catalog::LoadTableSchema` does. No rows are built, so a context that carries no catalog is + /// accepted here and refused by `Load` instead. + static Result> LoadForSchema( + const std::string& table_name, const GlobalSystemTableContext& context); + static Result> GetSupportedTableNames( const std::map& catalog_options); }; diff --git a/src/paimon/core/table/system/in_memory_system_table.cpp b/src/paimon/core/table/system/in_memory_system_table.cpp index d21d6fb4d..87fdd1d29 100644 --- a/src/paimon/core/table/system/in_memory_system_table.cpp +++ b/src/paimon/core/table/system/in_memory_system_table.cpp @@ -107,8 +107,8 @@ class InMemorySystemTableRead : public TableRead { } // namespace InMemorySystemTable::InMemorySystemTable(const std::shared_ptr& fs, - std::string table_path) - : SystemTable(fs, table_path), table_path_(std::move(table_path)) {} + std::string table_path, std::string name) + : SystemTable(fs, table_path, std::move(name)), table_path_(std::move(table_path)) {} Result> InMemorySystemTable::NewScan( std::unique_ptr /*context*/) const { diff --git a/src/paimon/core/table/system/in_memory_system_table.h b/src/paimon/core/table/system/in_memory_system_table.h index 90cd46d84..c72d7389e 100644 --- a/src/paimon/core/table/system/in_memory_system_table.h +++ b/src/paimon/core/table/system/in_memory_system_table.h @@ -36,7 +36,8 @@ class FileSystem; /// It provides the common singleton split scan and one-shot batch reader. class InMemorySystemTable : public SystemTable { public: - InMemorySystemTable(const std::shared_ptr& fs, std::string table_path); + InMemorySystemTable(const std::shared_ptr& fs, std::string table_path, + std::string name); Result> NewScan( std::unique_ptr /*context*/) const override; diff --git a/src/paimon/core/table/system/metadata_system_tables.cpp b/src/paimon/core/table/system/metadata_system_tables.cpp index e9e00ea42..1b5927004 100644 --- a/src/paimon/core/table/system/metadata_system_tables.cpp +++ b/src/paimon/core/table/system/metadata_system_tables.cpp @@ -507,11 +507,8 @@ Result> WriteColsValue( OptionsSystemTable::OptionsSystemTable(const std::shared_ptr& fs, std::string table_path, std::shared_ptr table_schema) - : InMemorySystemTable(fs, table_path), table_schema_(std::move(table_schema)) {} - -std::string OptionsSystemTable::Name() const { - return kName; -} + : InMemorySystemTable(fs, table_path, ComposeName(table_path, kName)), + table_schema_(std::move(table_schema)) {} Result> OptionsSystemTable::ArrowSchema() const { return arrow::schema({arrow::field("key", arrow::utf8(), /*nullable=*/false), @@ -533,13 +530,9 @@ Result> OptionsSystemTable::BuildRows() const { SnapshotsSystemTable::SnapshotsSystemTable(std::shared_ptr fs, std::string table_path, std::string branch) - : InMemorySystemTable(fs, table_path), + : InMemorySystemTable(fs, table_path, ComposeName(table_path, kName)), context_(CreateMetadataContext(std::move(fs), std::move(table_path), std::move(branch))) {} -std::string SnapshotsSystemTable::Name() const { - return kName; -} - Result> SnapshotsSystemTable::ArrowSchema() const { return arrow::schema({ arrow::field("snapshot_id", arrow::int64(), /*nullable=*/false), @@ -595,13 +588,9 @@ Result> SnapshotsSystemTable::BuildRows() const { SchemasSystemTable::SchemasSystemTable(std::shared_ptr fs, std::string table_path, std::string branch) - : InMemorySystemTable(fs, table_path), + : InMemorySystemTable(fs, table_path, ComposeName(table_path, kName)), context_(CreateMetadataContext(std::move(fs), std::move(table_path), std::move(branch))) {} -std::string SchemasSystemTable::Name() const { - return kName; -} - Result> SchemasSystemTable::ArrowSchema() const { return arrow::schema({ arrow::field("schema_id", arrow::int64(), /*nullable=*/false), @@ -651,13 +640,9 @@ Result> SchemasSystemTable::BuildRows() const { TagsSystemTable::TagsSystemTable(std::shared_ptr fs, std::string table_path, std::string branch) - : InMemorySystemTable(fs, table_path), + : InMemorySystemTable(fs, table_path, ComposeName(table_path, kName)), context_(CreateMetadataContext(std::move(fs), std::move(table_path), std::move(branch))) {} -std::string TagsSystemTable::Name() const { - return kName; -} - Result> TagsSystemTable::ArrowSchema() const { return arrow::schema({ arrow::field("tag_name", arrow::utf8(), /*nullable=*/false), @@ -701,13 +686,9 @@ Result> TagsSystemTable::BuildRows() const { BranchesSystemTable::BranchesSystemTable(std::shared_ptr fs, std::string table_path, std::string branch) - : InMemorySystemTable(fs, table_path), + : InMemorySystemTable(fs, table_path, ComposeName(table_path, kName)), context_(CreateMetadataContext(std::move(fs), std::move(table_path), std::move(branch))) {} -std::string BranchesSystemTable::Name() const { - return kName; -} - Result> BranchesSystemTable::ArrowSchema() const { return arrow::schema({ arrow::field("branch_name", arrow::utf8(), /*nullable=*/false), @@ -740,13 +721,9 @@ Result> BranchesSystemTable::BuildRows() const { ConsumersSystemTable::ConsumersSystemTable(std::shared_ptr fs, std::string table_path, std::string branch) - : InMemorySystemTable(fs, table_path), + : InMemorySystemTable(fs, table_path, ComposeName(table_path, kName)), context_(CreateMetadataContext(std::move(fs), std::move(table_path), std::move(branch))) {} -std::string ConsumersSystemTable::Name() const { - return kName; -} - Result> ConsumersSystemTable::ArrowSchema() const { return arrow::schema({ arrow::field("consumer_id", arrow::utf8(), /*nullable=*/false), @@ -775,14 +752,10 @@ ManifestsSystemTable::ManifestsSystemTable(std::shared_ptr fs, std:: std::string branch, std::shared_ptr table_schema, std::map options) - : InMemorySystemTable(fs, table_path), + : InMemorySystemTable(fs, table_path, ComposeName(table_path, kName)), context_(CreateMetadataContext(std::move(fs), std::move(table_path), std::move(branch), std::move(table_schema), std::move(options))) {} -std::string ManifestsSystemTable::Name() const { - return kName; -} - Result> ManifestsSystemTable::ArrowSchema() const { return arrow::schema({ arrow::field("file_name", arrow::utf8(), /*nullable=*/false), @@ -844,14 +817,10 @@ Result> ManifestsSystemTable::BuildRows() const { FilesSystemTable::FilesSystemTable(std::shared_ptr fs, std::string table_path, std::string branch, std::shared_ptr table_schema, std::map options) - : InMemorySystemTable(fs, table_path), + : InMemorySystemTable(fs, table_path, ComposeName(table_path, kName)), context_(CreateMetadataContext(std::move(fs), std::move(table_path), std::move(branch), std::move(table_schema), std::move(options))) {} -std::string FilesSystemTable::Name() const { - return kName; -} - Result> FilesSystemTable::ArrowSchema() const { return arrow::schema({ arrow::field("partition", arrow::utf8(), /*nullable=*/true), diff --git a/src/paimon/core/table/system/metadata_system_tables.h b/src/paimon/core/table/system/metadata_system_tables.h index ae2e5155b..1127e37ed 100644 --- a/src/paimon/core/table/system/metadata_system_tables.h +++ b/src/paimon/core/table/system/metadata_system_tables.h @@ -38,7 +38,6 @@ class OptionsSystemTable : public InMemorySystemTable { OptionsSystemTable(const std::shared_ptr& fs, std::string table_path, std::shared_ptr table_schema); - std::string Name() const override; Result> ArrowSchema() const override; Result> BuildRows() const override; @@ -63,7 +62,6 @@ class SnapshotsSystemTable : public InMemorySystemTable { SnapshotsSystemTable(std::shared_ptr fs, std::string table_path, std::string branch); - std::string Name() const override; Result> ArrowSchema() const override; Result> BuildRows() const override; @@ -78,7 +76,6 @@ class SchemasSystemTable : public InMemorySystemTable { SchemasSystemTable(std::shared_ptr fs, std::string table_path, std::string branch); - std::string Name() const override; Result> ArrowSchema() const override; Result> BuildRows() const override; @@ -93,7 +90,6 @@ class TagsSystemTable : public InMemorySystemTable { TagsSystemTable(std::shared_ptr fs, std::string table_path, std::string branch); - std::string Name() const override; Result> ArrowSchema() const override; Result> BuildRows() const override; @@ -108,7 +104,6 @@ class BranchesSystemTable : public InMemorySystemTable { BranchesSystemTable(std::shared_ptr fs, std::string table_path, std::string branch); - std::string Name() const override; Result> ArrowSchema() const override; Result> BuildRows() const override; @@ -124,7 +119,6 @@ class ConsumersSystemTable : public InMemorySystemTable { ConsumersSystemTable(std::shared_ptr fs, std::string table_path, std::string branch); - std::string Name() const override; Result> ArrowSchema() const override; Result> BuildRows() const override; @@ -141,7 +135,6 @@ class ManifestsSystemTable : public InMemorySystemTable { std::shared_ptr table_schema, std::map options); - std::string Name() const override; Result> ArrowSchema() const override; Result> BuildRows() const override; @@ -158,7 +151,6 @@ class FilesSystemTable : public InMemorySystemTable { std::shared_ptr table_schema, std::map options); - std::string Name() const override; Result> ArrowSchema() const override; Result> BuildRows() const override; diff --git a/src/paimon/core/table/system/read_optimized_system_table.cpp b/src/paimon/core/table/system/read_optimized_system_table.cpp index 65a48c048..2c4123f2f 100644 --- a/src/paimon/core/table/system/read_optimized_system_table.cpp +++ b/src/paimon/core/table/system/read_optimized_system_table.cpp @@ -40,15 +40,11 @@ ReadOptimizedSystemTable::ReadOptimizedSystemTable(const std::shared_ptr table_schema, std::map options) - : SystemTable(fs, table_path), + : SystemTable(fs, table_path, ComposeName(table_path, kName)), table_path_(std::move(table_path)), table_schema_(std::move(table_schema)), options_(std::move(options)) {} -std::string ReadOptimizedSystemTable::Name() const { - return kName; -} - Result> ReadOptimizedSystemTable::ArrowSchema() const { return DataField::ConvertDataFieldsToArrowSchema(table_schema_->Fields()); } @@ -74,6 +70,8 @@ Result> ReadOptimizedSystemTable::NewScan( .WithMemoryPool(context->GetMemoryPool()) .WithExecutor(context->GetExecutor()) // The view's own file system when the context names none, rather than the default backend. + // A `ScanContext` carries no scheme-to-identifier map, so unlike `CreateDataReadContext` + // below there is no map here to resolve through instead. .WithFileSystem(context->GetSpecificFileSystem() != nullptr ? context->GetSpecificFileSystem() : GetFileSystem()) diff --git a/src/paimon/core/table/system/read_optimized_system_table.h b/src/paimon/core/table/system/read_optimized_system_table.h index 7f92e04e2..82fec3ef4 100644 --- a/src/paimon/core/table/system/read_optimized_system_table.h +++ b/src/paimon/core/table/system/read_optimized_system_table.h @@ -38,7 +38,6 @@ class ReadOptimizedSystemTable : public SystemTable { std::shared_ptr table_schema, std::map options); - std::string Name() const override; Result> ArrowSchema() const override; Result> NewScan(std::unique_ptr context) const override; Result> NewRead(std::unique_ptr context) const override; diff --git a/src/paimon/core/table/system/system_table.cpp b/src/paimon/core/table/system/system_table.cpp index 0161ad6b7..7a8146534 100644 --- a/src/paimon/core/table/system/system_table.cpp +++ b/src/paimon/core/table/system/system_table.cpp @@ -42,6 +42,7 @@ #include "paimon/core/utils/branch_manager.h" #include "paimon/file_store_commit.h" #include "paimon/file_store_write.h" +#include "paimon/predicate/predicate_utils.h" #include "paimon/read_context.h" #include "paimon/status.h" #include "paimon/write_context.h" @@ -182,15 +183,27 @@ std::optional FindSystemTableFactory(const std::string& syst } // namespace -SystemTable::SystemTable(const std::shared_ptr& fs, const std::string& table_path) +SystemTable::SystemTable(const std::shared_ptr& fs, const std::string& table_path, + std::string name) : Table(fs, table_path, /*schema=*/nullptr, - Identifier(Identifier::kUnknownDatabase, "system_table"), /*uuid=*/"") {} + Identifier(Identifier::kUnknownDatabase, "system_table"), /*uuid=*/""), + name_(std::move(name)) {} + +std::string SystemTable::ComposeName(const std::string& table_path, const std::string& kind) { + return PathUtil::GetName(table_path) + Catalog::SYSTEM_TABLE_SPLITTER + kind; +} Status SystemTable::CheckNoSearch(const std::shared_ptr& context) { if (context->HasVectorOrFullTextSearch()) { return Status::NotImplemented( "VectorSearch or FullTextSearch does not support system-table reads yet"); } + // A `FullTextMatch` predicate is not a TopK request, so the check above does not see it. The + // factory refuses one for a system table with the same message. An in-memory view serves rows + // of its own and never evaluates a predicate, so without this it answers with all of them. + if (PredicateUtils::ContainsFullTextMatch(context->GetPredicate())) { + return Status::NotImplemented("FullTextMatch predicate execution is not implemented yet"); + } return Status::OK(); } @@ -282,7 +295,7 @@ Result> SystemTableLoader::LoadFromPath( // The warehouse is the grandparent of the sys/
path context.warehouse = PathUtil::GetParentDirPath(PathUtil::GetParentDirPath(path)); context.catalog_options = dynamic_options; - // Note: context.catalog is intentionally left as nullptr here. + // Note: context.catalog is intentionally left empty here. // Global tables loaded from path do not have a Catalog reference and // cannot enumerate databases/tables. Only tables that don't require // catalog enumeration (e.g. catalog_options) will work in this path. diff --git a/src/paimon/core/table/system/system_table.h b/src/paimon/core/table/system/system_table.h index 4c235d3cf..640672236 100644 --- a/src/paimon/core/table/system/system_table.h +++ b/src/paimon/core/table/system/system_table.h @@ -50,21 +50,29 @@ struct SystemTablePath { bool is_global = false; }; -/// Base class for table-scoped system tables such as `T$options` and `T$snapshots`: the read-only -/// view a `T$system` identifier names, which exposes a schema of its own and serves scans and -/// reads from the data table it belongs to. `Name()` - which each kind answers for itself - is -/// its full name as well: the identifier it is opened by belongs to the data table underneath. +/// Base class for the table-scoped system tables such as `T$options` and `T$snapshots`. A +/// `T$system` identifier names a read-only view that exposes a schema of its own and serves scans +/// and reads from the data table it belongs to. +/// +/// A system table of a data table answers `$` for `Name()`. One of the `sys` +/// database answers the bare kind name. `FullName()` returns the same as `Name()` in both cases, +/// which is what Java Paimon's `Table.fullName()` defaults to. class SystemTable : public Table, public std::enable_shared_from_this { public: + std::string Name() const override { + return name_; + } + std::string FullName() const override { - return Name(); + return name_; } - /// The read-only schema this system table exposes, which is not the data table's schema. + /// Returns the read-only schema this system table exposes. It is not the data table's schema. virtual Result> ArrowSchema() const = 0; - /// A system table is read-only: copying it, writing it and committing for it are refused - /// rather than routed to the data table underneath, which its own `Table` serves. + /// A system table is read-only, so copying it, writing it and committing for it are refused + /// instead of routed to the data table underneath. That table serves them through its own + /// `Table`. Result> Copy( const std::map& dynamic_options) const override; @@ -77,19 +85,29 @@ class SystemTable : public Table, public std::enable_shared_from_this& fs, const std::string& table_path); - - /// Refuses a read whose context asks for a search: each system table builds a context of its - /// own for the data table underneath, which carries no search, so answering one would return - /// plain rows and quietly drop the search and its top K. + /// @param name The name this table answers with: `ComposeName(table_path, kind)` for a + /// table-scoped system table, the bare kind name for one of the `sys` database. + SystemTable(const std::shared_ptr& fs, const std::string& table_path, + std::string name); + + /// Composes the name of a table-scoped system table. Java Paimon's `OptionsTable.name()` + /// composes it the same way: `location.getName() + SYSTEM_TABLE_SPLITTER + OPTIONS`. + static std::string ComposeName(const std::string& table_path, const std::string& kind); + + /// Refuses a read whose context asks for a search: a root TopK request or a `FullTextMatch` + /// predicate. Neither is evaluated on the rows a system table serves, so answering one + /// would return plain rows and quietly drop the search and its top K. static Status CheckNoSearch(const std::shared_ptr& context); - /// Loaders call this once the object exists, because `ArrowSchema()` is virtual and cannot run - /// from a constructor: it fills in this table's own read-only `SystemTableSchema`. + /// Fills in this table's own read-only `SystemTableSchema`. A loader calls it once the object + /// exists, because `ArrowSchema()` is virtual and cannot run from a constructor. Status AdoptOwnSchema(); friend class SystemTableLoader; friend class GlobalSystemTableLoader; + + private: + std::string name_; }; /// Loads system table implementations from parsed table identifiers or table paths. diff --git a/src/paimon/core/table/system/system_table_test.cpp b/src/paimon/core/table/system/system_table_test.cpp index f1f0cbfb1..d6333475b 100644 --- a/src/paimon/core/table/system/system_table_test.cpp +++ b/src/paimon/core/table/system/system_table_test.cpp @@ -38,6 +38,7 @@ #include "paimon/core/table/system/audit_log_system_table.h" #include "paimon/core/table/system/binlog_system_table.h" #include "paimon/core/table/system/global_system_tables.h" +#include "paimon/core/table/system/metadata_system_tables.h" #include "paimon/core/table/system/read_optimized_system_table.h" #include "paimon/core/table/system/system_table_scan.h" #include "paimon/defs.h" @@ -45,6 +46,9 @@ #include "paimon/fs/file_system_factory.h" #include "paimon/memory/memory_pool.h" #include "paimon/metrics.h" +#include "paimon/predicate/full_text_match.h" +#include "paimon/predicate/predicate.h" +#include "paimon/predicate/predicate_builder.h" #include "paimon/predicate/vector_search.h" #include "paimon/read_context.h" #include "paimon/reader/batch_reader.h" @@ -222,7 +226,9 @@ TEST(SystemTableTest, TestReadOptimizedSystemTableRegistration) { SystemTableLoader::Load(ReadOptimizedSystemTable::kName, /*fs=*/nullptr, "/tmp/table", table_schema, /*dynamic_options=*/{})); - ASSERT_EQ(system_table->Name(), ReadOptimizedSystemTable::kName); + // The name carries the data table it belongs to, the way Java Paimon's + // `ReadOptimizedTable.name()` does. + ASSERT_EQ(system_table->Name(), "table$ro"); ASSERT_OK_AND_ASSIGN(std::shared_ptr arrow_schema, system_table->ArrowSchema()); ASSERT_EQ(arrow_schema->field_names(), (std::vector{"pk", "v"})); @@ -360,6 +366,39 @@ TEST(SystemTableTest, TestNewReadRefusesASearch) { "VectorSearch or FullTextSearch does not support system-table reads yet"); } +// `SetPredicate` carries a search too, as a `FullTextMatch` leaf. An in-memory view reads rows +// of its own and never evaluates a predicate, so without the check it would answer with all of +// them where `TableRead::Create` refuses the same request. +TEST(SystemTableTest, TestNewReadRefusesAFullTextMatchPredicate) { + std::map options = {{Options::FILE_SYSTEM, "local"}, + {Options::FILE_FORMAT, "orc"}}; + ASSERT_OK_AND_ASSIGN(std::shared_ptr table_schema, + CreateTableSchemaForTest(options)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr in_memory, + SystemTableLoader::Load(OptionsSystemTable::kName, /*fs=*/nullptr, + "/tmp/table", table_schema, + /*dynamic_options=*/{})); + ReadOptimizedSystemTable read_optimized(/*fs=*/nullptr, "/tmp/table", table_schema, options); + + auto match = std::make_shared("pk", R"({"match":{"query":"apple"}})"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr predicate, + PredicateBuilder::FullTextMatch(/*field_index=*/0, match)); + + auto make_context = [&predicate] { + ReadContextBuilder builder("/tmp/table"); + builder.SetPredicate(predicate); + return builder.Finish(); + }; + + ASSERT_OK_AND_ASSIGN(std::unique_ptr in_memory_context, make_context()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_optimized_context, make_context()); + + ASSERT_NOK_WITH_MSG(in_memory->NewRead(std::move(in_memory_context)), + "FullTextMatch predicate execution is not implemented yet"); + ASSERT_NOK_WITH_MSG(read_optimized.NewRead(std::move(read_optimized_context)), + "FullTextMatch predicate execution is not implemented yet"); +} + namespace { /// A catalog that serves the files of every table through `GetTableFileSystem`, like one @@ -392,21 +431,25 @@ TEST(SystemTableTest, TestGlobalPartitionsUsesTheFileSystemOfEachTable) { std::unique_ptr dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); - PerTableFileSystemCatalog catalog(core_options.GetFileSystem(), dir->Str(), options); - ASSERT_OK(catalog.CreateDatabase("db1", options, /*ignore_if_exists=*/true)); + // Held by a `shared_ptr` because a global system table keeps shared ownership of the catalog + // it reads its rows through. + std::shared_ptr catalog = + std::make_shared(core_options.GetFileSystem(), dir->Str(), + options); + ASSERT_OK(catalog->CreateDatabase("db1", options, /*ignore_if_exists=*/true)); arrow::Schema typed_schema( {arrow::field("dt", arrow::utf8()), arrow::field("v", arrow::int32(), /*nullable=*/true)}); ::ArrowSchema c_schema; ASSERT_TRUE(arrow::ExportSchema(typed_schema, &c_schema).ok()); - Status created = catalog.CreateTable(Identifier("db1", "t1"), &c_schema, - /*partition_keys=*/{"dt"}, /*primary_keys=*/{}, options, - /*ignore_if_exists=*/false); + Status created = catalog->CreateTable(Identifier("db1", "t1"), &c_schema, + /*partition_keys=*/{"dt"}, /*primary_keys=*/{}, options, + /*ignore_if_exists=*/false); ArrowSchemaRelease(&c_schema); ASSERT_OK(created); GlobalSystemTableContext context; - context.catalog = &catalog; + context.catalog = catalog; context.fs = core_options.GetFileSystem(); context.warehouse = dir->Str(); context.catalog_options = options; @@ -416,16 +459,16 @@ TEST(SystemTableTest, TestGlobalPartitionsUsesTheFileSystemOfEachTable) { // still the ones asked for before its files are listed. ASSERT_OK_AND_ASSIGN(std::vector rows, partitions.BuildRows()); ASSERT_TRUE(rows.empty()); - ASSERT_EQ((std::vector{"db1.t1"}), catalog.requested); + ASSERT_EQ((std::vector{"db1.t1"}), catalog->requested); // A table whose credentials cannot be issued fails the query, so that a partial result // is never mistaken for the whole one. - catalog.failure = Status::Invalid("no credentials for the table"); + catalog->failure = Status::Invalid("no credentials for the table"); ASSERT_NOK_WITH_MSG(partitions.BuildRows(), "no credentials for the table"); // A table that went away between being listed and being read is skipped instead, like // one whose location has already been removed. - catalog.failure = Status::NotExist("table dropped"); + catalog->failure = Status::NotExist("table dropped"); ASSERT_OK_AND_ASSIGN(std::vector skipped, partitions.BuildRows()); ASSERT_TRUE(skipped.empty()); } diff --git a/src/paimon/core/table/table_test.cpp b/src/paimon/core/table/table_test.cpp index 055fc5995..5100dfd78 100644 --- a/src/paimon/core/table/table_test.cpp +++ b/src/paimon/core/table/table_test.cpp @@ -75,7 +75,8 @@ Result> CreateManagedTable( return FileStoreTable::Create( file_system, path, checked_pointer_cast(std::shared_ptr(std::move(created))), identifier, - /*uuid=*/"", branch); + /*uuid=*/"", branch, /*schema_manager=*/nullptr, /*snapshot_manager=*/nullptr, + /*fs_scheme_map=*/{}, /*dynamic_options=*/{}); } /// Writes two rows through the table's own write and commit, so a scan below has data to find. @@ -243,7 +244,8 @@ TEST(TableTest, TestACopyHandsItsDynamicOptionsToTheFallbackReader) { FileStoreTable::Create( fs, path, checked_pointer_cast(std::shared_ptr(std::move(created))), - identifier, /*uuid=*/"", /*branch=*/"main")); + identifier, /*uuid=*/"", /*branch=*/"main", /*schema_manager=*/nullptr, + /*snapshot_manager=*/nullptr, /*fs_scheme_map=*/{}, /*dynamic_options=*/{})); // The copy turns the sequence-number column on, and a read projects it through a context // that carries no options of its own. The main reader serves the copy's merged options, so @@ -286,7 +288,8 @@ TEST(TableTest, TestACopyOfACopyServesTheLatestValueOfARepeatedDynamicOption) { FileStoreTable::Create( fs, path, checked_pointer_cast(std::shared_ptr(std::move(created))), - identifier, /*uuid=*/"", /*branch=*/"main")); + identifier, /*uuid=*/"", /*branch=*/"main", /*schema_manager=*/nullptr, + /*snapshot_manager=*/nullptr, /*fs_scheme_map=*/{}, /*dynamic_options=*/{})); // The option is turned off and then on again by the copy the read serves, so the copy's // schema carries the latest value. The fallback branch's schema does not carry the option @@ -328,7 +331,8 @@ TEST(TableTest, TestAFallbackReaderServesItsBranchsOwnSchema) { FileStoreTable::Create( fs, path, checked_pointer_cast(std::shared_ptr(std::move(main_created))), - identifier, /*uuid=*/"", /*branch=*/"main")); + identifier, /*uuid=*/"", /*branch=*/"main", /*schema_manager=*/nullptr, + /*snapshot_manager=*/nullptr, /*fs_scheme_map=*/{}, /*dynamic_options=*/{})); SchemaManager fallback_schema_manager(fs, path, "fb"); ASSERT_OK_AND_ASSIGN( @@ -341,7 +345,8 @@ TEST(TableTest, TestAFallbackReaderServesItsBranchsOwnSchema) { FileStoreTable::Create( fs, path, checked_pointer_cast(std::shared_ptr(std::move(fallback_created))), - identifier, /*uuid=*/"", /*branch=*/"fb")); + identifier, /*uuid=*/"", /*branch=*/"fb", /*schema_manager=*/nullptr, + /*snapshot_manager=*/nullptr, /*fs_scheme_map=*/{}, /*dynamic_options=*/{})); WriteAndCommitRow(fallback, path, /*id=*/1, /*v=*/10); WriteAndCommitRow(fallback, path, /*id=*/1, /*v=*/20); @@ -679,7 +684,11 @@ TEST(TableTest, TestMainBranchAliasResolvesToTheMainBranch) { ASSERT_OK_AND_ASSIGN(std::string branch, main_identifier.GetBranchNameOrDefault()); ASSERT_OK_AND_ASSIGN(std::shared_ptr table, FileStoreTable::Create(fs, dir->Str(), schema, main_identifier, - /*uuid=*/"", branch)); + /*uuid=*/"", branch, + /*schema_manager=*/nullptr, + /*snapshot_manager=*/nullptr, + /*fs_scheme_map=*/{}, + /*dynamic_options=*/{})); EXPECT_EQ("main", table->CurrentBranch()); } } @@ -1112,7 +1121,10 @@ TEST(TableTest, TestCreateFailsWhenTheSchemaOptionsDoNotParse) { /*partition_keys=*/{}, /*primary_keys=*/{}, {{Options::TARGET_FILE_ROW_NUM, "0"}})); auto schema = checked_pointer_cast(std::shared_ptr(std::move(created))); ASSERT_NOK_WITH_MSG( - FileStoreTable::Create(dir->GetFileSystem(), dir->Str(), schema, Identifier("db", "tbl")), + FileStoreTable::Create(dir->GetFileSystem(), dir->Str(), schema, Identifier("db", "tbl"), + /*uuid=*/"", /*branch=*/BranchManager::DEFAULT_MAIN_BRANCH, + /*schema_manager=*/nullptr, /*snapshot_manager=*/nullptr, + /*fs_scheme_map=*/{}, /*dynamic_options=*/{}), "target-file-row-num should be at least 1"); } diff --git a/src/paimon/rest/rest_catalog.cpp b/src/paimon/rest/rest_catalog.cpp index fec764db3..0752af6f7 100644 --- a/src/paimon/rest/rest_catalog.cpp +++ b/src/paimon/rest/rest_catalog.cpp @@ -383,13 +383,15 @@ Result> RestCatalog::LoadTableSchema(const Identifier& i if (!supported) { return Status::NotExist(fmt::format("{} not exist", identifier.ToString())); } + // Only the schema is read here and the table does not outlive the call, so it needs no + // catalog and this one hands over no ownership of itself. GlobalSystemTableContext context; - context.catalog = this; context.fs = fs_; context.warehouse = warehouse_; context.catalog_options = api_->GetMergedOptions(); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr system_table, - GlobalSystemTableLoader::Load(identifier.GetObjectName(), context)); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr system_table, + GlobalSystemTableLoader::LoadForSchema(identifier.GetObjectName(), context)); return system_table->LatestSchema(); } PAIMON_ASSIGN_OR_RAISE(bool is_system_table, identifier.IsSystemTable()); @@ -444,18 +446,17 @@ Result> RestCatalog::GetTable(const Identifier& identifie if (!supported) { return Status::NotExist(fmt::format("{} not exist", identifier.ToString())); } - GlobalSystemTableContext context; - context.catalog = this; - // A global system table builds its rows lazily through the catalog above, so the table - // handed back has to keep it alive or the pointer dangles at the first `BuildRows`. A - // catalog not owned by a `std::shared_ptr` cannot be kept, so it is refused by name. + // A global system table builds its rows lazily through this catalog, so the table handed + // back has to keep it alive or the pointer dangles at the first `BuildRows`. A catalog + // not owned by a `std::shared_ptr` cannot be kept, so it is refused by name. std::shared_ptr self = weak_from_this().lock(); if (self == nullptr) { return Status::Invalid( "a global system table retains its catalog, so the catalog must be owned by " "a std::shared_ptr before it is asked for one"); } - context.catalog_retainer = std::move(self); + GlobalSystemTableContext context; + context.catalog = std::move(self); context.fs = fs_; context.warehouse = warehouse_; context.catalog_options = api_->GetMergedOptions(); @@ -496,7 +497,9 @@ Result> RestCatalog::GetTable(const Identifier& identifie PAIMON_ASSIGN_OR_RAISE(std::optional branch, identifier.GetBranchName()); return FileStoreTable::Create( table_file_system, table_path, schema, identifier, uuid, - NormalizeBranch(std::move(branch)).value_or(Identifier::kDefaultMainBranch)); + NormalizeBranch(std::move(branch)).value_or(Identifier::kDefaultMainBranch), + /*schema_manager=*/nullptr, /*snapshot_manager=*/nullptr, /*fs_scheme_map=*/{}, + /*dynamic_options=*/{}); } std::string RestCatalog::GetRootPath() const { diff --git a/src/paimon/rest/rest_catalog_test.cpp b/src/paimon/rest/rest_catalog_test.cpp index 5d418fc63..cc197b45d 100644 --- a/src/paimon/rest/rest_catalog_test.cpp +++ b/src/paimon/rest/rest_catalog_test.cpp @@ -743,7 +743,7 @@ TEST_F(RestCatalogTest, RecreatedTableLoadsNewCredentialsBeforeOldTokenExpires) } TEST_F(RestCatalogTest, CatalogFactoryMetastoreDispatch) { - ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, Catalog::Create(kWarehouse, options_)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr catalog, Catalog::Create(kWarehouse, options_)); ASSERT_OK_AND_ASSIGN(std::vector databases, catalog->ListDatabases()); ASSERT_TRUE(databases.empty()); options_[CatalogOptions::METASTORE] = "something-else"; @@ -1003,13 +1003,13 @@ TEST_F(RestCatalogTest, SystemTableSchema) { requests_before = state_->get_table_requests; } ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, catalog->GetTable(system_identifier)); - // A system table comes back as the read-only `SystemTable` it is, answering for itself - // with the kind it serves rather than the identifier it was opened by. + // A system table comes back as the read-only `SystemTable` it is, naming itself after the + // data table it belongs to. std::shared_ptr system_table = std::dynamic_pointer_cast(table); ASSERT_NE(nullptr, system_table); - ASSERT_EQ("options", system_table->Name()); + ASSERT_EQ("t1$options", system_table->Name()); ASSERT_EQ((std::vector{"key", "value"}), table->LatestSchema()->FieldNames()); - ASSERT_EQ("options", table->Uuid()); + ASSERT_EQ("t1$options", table->Uuid()); { std::lock_guard lock(state_->mutex); ASSERT_EQ(requests_before + 1, state_->get_table_requests); @@ -1511,6 +1511,34 @@ TEST_F(RestCatalogTest, GlobalSystemTableRetainsItsCatalog) { ASSERT_GT(array->length(), 0); } +TEST_F(RestCatalogTest, GlobalSystemTableWorksThroughThePublicCatalogFactory) { + // `Catalog::Create()` is the factory a caller outside paimon uses, and it hands back a + // `Catalog` rather than a `RestCatalog`. A global system table keeps its catalog alive + // through that pointer, so this path has to work too - a catalog whose shared ownership + // was not set up would fail here with the refusal naming the missing ownership. + ASSERT_OK_AND_ASSIGN(std::shared_ptr catalog, Catalog::Create(kWarehouse, options_)); + ASSERT_OK(catalog->CreateDatabase("db1", {}, /*ignore_if_exists=*/false)); + ASSERT_OK(CreateSampleTable(catalog.get(), Identifier("db1", "t1"))); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, + catalog->GetTable(Identifier("sys", "tables"))); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, + ScanContextBuilder("sys/tables").Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, table->NewScan(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); + ASSERT_EQ(plan->Splits().size(), 1); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, + ReadContextBuilder("sys/tables").Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read, table->NewRead(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_reader, + read->CreateReader(plan->Splits())); + ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch batch, batch_reader->NextBatch()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr array, + ReadResultCollector::GetArray(std::move(batch))); + ASSERT_TRUE(array); + ASSERT_GT(array->length(), 0); +} + TEST_F(RestCatalogTest, ListSnapshots) { ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, CreateRestCatalog()); ASSERT_OK(catalog->CreateDatabase("db1", {}, /*ignore_if_exists=*/false)); diff --git a/src/paimon/testing/mock/mock_catalog.h b/src/paimon/testing/mock/mock_catalog.h index 57a6e8d98..c94284eb2 100644 --- a/src/paimon/testing/mock/mock_catalog.h +++ b/src/paimon/testing/mock/mock_catalog.h @@ -101,7 +101,9 @@ class MockVersionManagedCatalog : public Catalog, public VersionManagedCatalog { // this mock's `GetTableLocation` is the unsupported stub. return FileStoreTable::Create( table_file_system_ != nullptr ? table_file_system_ : file_system_, - /*root_path=*/"", table_schema_, identifier, table_uuid_); + /*root_path=*/"", table_schema_, identifier, table_uuid_, + /*branch=*/BranchManager::DEFAULT_MAIN_BRANCH, /*schema_manager=*/nullptr, + /*snapshot_manager=*/nullptr, /*fs_scheme_map=*/{}, /*dynamic_options=*/{}); } void SetTableSchema(const std::shared_ptr& table_schema) { diff --git a/test/inte/read_inte_test.cpp b/test/inte/read_inte_test.cpp index 2c24607a2..ef8e5c853 100644 --- a/test/inte/read_inte_test.cpp +++ b/test/inte/read_inte_test.cpp @@ -4173,8 +4173,9 @@ TEST_P(ReadInteTest, TestSpecificFs) { namespace { Result> ReadGlobalSystemTable( - const std::string& table_name, Catalog* catalog, const std::shared_ptr& fs, - const std::string& warehouse, const std::map& options) { + const std::string& table_name, const std::shared_ptr& catalog, + const std::shared_ptr& fs, const std::string& warehouse, + const std::map& options) { GlobalSystemTableContext ctx; ctx.catalog = catalog; ctx.fs = fs; @@ -4229,8 +4230,8 @@ TEST(SystemTableReadInteTest, TestReadGlobalCatalogOptions) { std::string warehouse = PathUtil::JoinPath(dir->Str(), "warehouse"); ASSERT_OK_AND_ASSIGN(auto catalog, Catalog::Create(warehouse, options)); ASSERT_OK_AND_ASSIGN(auto result, - ReadGlobalSystemTable("catalog_options", catalog.get(), - catalog->GetFileSystem(), warehouse, options)); + ReadGlobalSystemTable("catalog_options", catalog, catalog->GetFileSystem(), + warehouse, options)); auto struct_array = SingleStructChunk(result); ASSERT_TRUE(struct_array); auto key_array = std::dynamic_pointer_cast(struct_array->field(0)); @@ -4301,7 +4302,7 @@ TEST(SystemTableReadInteTest, TestReadGlobalAllTableOptions) { // Directly test BuildRows { GlobalSystemTableContext ctx; - ctx.catalog = catalog.get(); + ctx.catalog = catalog; ctx.fs = catalog->GetFileSystem(); ctx.warehouse = warehouse; ctx.catalog_options = options; @@ -4310,9 +4311,9 @@ TEST(SystemTableReadInteTest, TestReadGlobalAllTableOptions) { ASSERT_GT(rows.size(), 0) << "BuildRows returned empty, expected at least 1 row"; } - ASSERT_OK_AND_ASSIGN(auto result, - ReadGlobalSystemTable("all_table_options", catalog.get(), - catalog->GetFileSystem(), warehouse, options)); + ASSERT_OK_AND_ASSIGN( + auto result, ReadGlobalSystemTable("all_table_options", catalog, catalog->GetFileSystem(), + warehouse, options)); auto struct_array = SingleStructChunk(result); ASSERT_TRUE(struct_array); ASSERT_GE(struct_array->length(), 1) << "result has " << struct_array->length() << " rows"; @@ -4392,7 +4393,7 @@ TEST(SystemTableReadInteTest, TestReadGlobalTables) { ArrowSchemaRelease(&no_pk_schema); ASSERT_OK_AND_ASSIGN(auto result, - ReadGlobalSystemTable("tables", catalog.get(), fs, warehouse, options)); + ReadGlobalSystemTable("tables", catalog, fs, warehouse, options)); auto struct_array = SingleStructChunk(result); ASSERT_TRUE(struct_array); ASSERT_GE(struct_array->length(), 1); @@ -4490,8 +4491,8 @@ TEST(SystemTableReadInteTest, TestReadGlobalPartitions) { ASSERT_OK_AND_ASSIGN(auto catalog, Catalog::Create(warehouse, options)); auto fs = catalog->GetFileSystem(); - ASSERT_OK_AND_ASSIGN(auto part_result, ReadGlobalSystemTable("partitions", catalog.get(), fs, - warehouse, options)); + ASSERT_OK_AND_ASSIGN(auto part_result, + ReadGlobalSystemTable("partitions", catalog, fs, warehouse, options)); auto struct_array = SingleStructChunk(part_result); ASSERT_TRUE(struct_array); ASSERT_EQ(StructFieldNames(struct_array), @@ -4542,7 +4543,7 @@ TEST(SystemTableReadInteTest, TestGlobalSystemTablesPropagateCorruptSchema) { /*overwrite=*/true)); for (const std::string system_table : {"all_table_options", "tables", "partitions"}) { - ASSERT_NOK(ReadGlobalSystemTable(system_table, catalog.get(), fs, warehouse, options)); + ASSERT_NOK(ReadGlobalSystemTable(system_table, catalog, fs, warehouse, options)); } } @@ -4578,7 +4579,7 @@ TEST(SystemTableReadInteTest, TestPartitionsSystemTablePropagatesCorruptSnapshot ASSERT_OK(fs->WriteFile(PathUtil::JoinPath(table_path, "snapshot/snapshot-1"), "{invalid-json", /*overwrite=*/true)); - ASSERT_NOK(ReadGlobalSystemTable("partitions", catalog.get(), fs, warehouse, options)); + ASSERT_NOK(ReadGlobalSystemTable("partitions", catalog, fs, warehouse, options)); } TEST(SystemTableReadInteTest, TestPartitionsSystemTablePropagatesCorruptManifest) { @@ -4617,7 +4618,7 @@ TEST(SystemTableReadInteTest, TestPartitionsSystemTablePropagatesCorruptManifest PathUtil::JoinPath(table_path, "manifest/" + snapshot->BaseManifestList()), "corrupt", /*overwrite=*/true)); - ASSERT_NOK(ReadGlobalSystemTable("partitions", catalog.get(), fs, warehouse, options)); + ASSERT_NOK(ReadGlobalSystemTable("partitions", catalog, fs, warehouse, options)); } } // namespace paimon::test diff --git a/test/inte/realtime_write_inte_test.cpp b/test/inte/realtime_write_inte_test.cpp index 5de6c9fa7..7111f5302 100644 --- a/test/inte/realtime_write_inte_test.cpp +++ b/test/inte/realtime_write_inte_test.cpp @@ -504,7 +504,7 @@ class RealtimeWriteInteTest : public ::testing::Test { void CreateTable(const std::vector& partition_keys) const { auto c_schema = std::make_unique(); ASSERT_TRUE(arrow::ExportSchema(*schema_, c_schema.get()).ok()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, + ASSERT_OK_AND_ASSIGN(std::shared_ptr catalog, Catalog::Create(dir_->Str(), options_)); ASSERT_OK(catalog->CreateDatabase("foo", {}, /*ignore_if_exists=*/false)); ASSERT_OK(catalog->CreateTable(Identifier("foo", "bar"), c_schema.get(), partition_keys, @@ -516,7 +516,7 @@ class RealtimeWriteInteTest : public ::testing::Test { const std::vector& primary_keys = {"id"}) const { auto c_schema = std::make_unique(); ASSERT_TRUE(arrow::ExportSchema(*schema_, c_schema.get()).ok()); - ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, + ASSERT_OK_AND_ASSIGN(std::shared_ptr catalog, Catalog::Create(dir_->Str(), options_)); ASSERT_OK(catalog->CreateDatabase("foo", {}, /*ignore_if_exists=*/false)); std::vector table_primary_keys = partition_keys; From 303751242db76a9c876fcf5aafd366a4c6befcdb Mon Sep 17 00:00:00 2001 From: "yonghao.fyh" Date: Tue, 27 Oct 2026 16:51:21 +0800 Subject: [PATCH 17/17] fix --- docs/source/user_guide/catalog.rst | 2 +- .../core/operation/file_store_commit.cpp | 16 ++++++++++ .../core/operation/file_store_write.cpp | 11 +++++++ src/paimon/core/schema/schema_validation.cpp | 5 +++ src/paimon/core/table/file_store_table.cpp | 32 +++---------------- src/paimon/core/table/source/table_read.cpp | 10 ++++++ src/paimon/core/table/source/table_scan.cpp | 10 ++++++ src/paimon/core/table/table_test.cpp | 7 ++++ 8 files changed, 65 insertions(+), 28 deletions(-) diff --git a/docs/source/user_guide/catalog.rst b/docs/source/user_guide/catalog.rst index 599b4ffce..ca593e38e 100644 --- a/docs/source/user_guide/catalog.rst +++ b/docs/source/user_guide/catalog.rst @@ -134,7 +134,7 @@ without an atomic snapshot-file rename: } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr commit, - paimon::FileStoreCommit::Create(std::move(context))); + table->NewCommit(std::move(context))); PAIMON_RETURN_NOT_OK(commit->Commit(commit_messages)); Configure native writers with the same catalog and identifier. They load the diff --git a/src/paimon/core/operation/file_store_commit.cpp b/src/paimon/core/operation/file_store_commit.cpp index 4e0747437..9b6c3975c 100644 --- a/src/paimon/core/operation/file_store_commit.cpp +++ b/src/paimon/core/operation/file_store_commit.cpp @@ -18,3 +18,19 @@ */ #include "paimon/file_store_commit.h" + +#include "paimon/catalog/table.h" +#include "paimon/commit_context.h" +#include "paimon/core/table/file_store_table_factory.h" +#include "paimon/core/utils/context_utils.h" + +namespace paimon { + +Result> FileStoreCommit::Create( + std::unique_ptr ctx) { + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(ctx, "commit")); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, FileStoreTableFactory::Create(*ctx)); + return table->NewCommit(std::move(ctx)); +} + +} // namespace paimon diff --git a/src/paimon/core/operation/file_store_write.cpp b/src/paimon/core/operation/file_store_write.cpp index c1b889473..ea73b2246 100644 --- a/src/paimon/core/operation/file_store_write.cpp +++ b/src/paimon/core/operation/file_store_write.cpp @@ -18,8 +18,19 @@ #include "paimon/file_store_write.h" +#include "paimon/catalog/table.h" +#include "paimon/core/table/file_store_table_factory.h" +#include "paimon/core/utils/context_utils.h" +#include "paimon/write_context.h" + namespace paimon { +Result> FileStoreWrite::Create(std::unique_ptr ctx) { + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(ctx, "write")); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, FileStoreTableFactory::Create(*ctx)); + return table->NewWrite(std::move(ctx)); +} + Result> FileStoreWrite::PrepareCommitWithProgress(int64_t) { return Status::Invalid("prepare commit with progress requires a real-time writer"); } diff --git a/src/paimon/core/schema/schema_validation.cpp b/src/paimon/core/schema/schema_validation.cpp index 5d112417f..c799c24f8 100644 --- a/src/paimon/core/schema/schema_validation.cpp +++ b/src/paimon/core/schema/schema_validation.cpp @@ -85,6 +85,11 @@ constexpr char kDeletionVectorsMergeOnRead[] = "deletion-vectors.merge-on-read"; /// into a table. Unlike the Java set, `merge-engine`, `bucket`, `row-tracking.enabled` and /// `data-evolution.enabled` stay servable as dynamic options here, and `type`/`branch` are dropped /// by `Copy` before this point. +// TODO(yonghao.fyh): refuse the rest of Java's immutable set, which this list does not cover yet: +// `blob-descriptor-field` (only its deprecated `blob.stored-descriptor-fields` fallback is above), +// `data-file.path-directory`, `sequence.snapshot-ordering`, `rowkind.field`, `primary-key`, +// `primary-key.nullable`, `partition`, `dynamic-bucket.initial-buckets`, `video-frame-field`, and +// the four servable keys named above, should the C++ side stop serving them as dynamic options. const char* const kImmutableOptions[] = { Options::BUCKET_KEY, Options::BUCKET_FUNCTION_TYPE, diff --git a/src/paimon/core/table/file_store_table.cpp b/src/paimon/core/table/file_store_table.cpp index 3945de5b7..da86a590e 100644 --- a/src/paimon/core/table/file_store_table.cpp +++ b/src/paimon/core/table/file_store_table.cpp @@ -55,7 +55,6 @@ #include "paimon/core/snapshot.h" #include "paimon/core/table/append_only_file_store_table.h" #include "paimon/core/table/file_store.h" -#include "paimon/core/table/file_store_table_factory.h" #include "paimon/core/table/primary_key_file_store_table.h" #include "paimon/core/table/source/append_only_table_read.h" #include "paimon/core/table/source/data_evolution_batch_scan.h" @@ -210,9 +209,13 @@ Result> FileStoreTable::SwitchToBranch(const std::string& branch_schema->Options(), std::static_pointer_cast(schema_)->Options()))) ? nullptr : schema_manager; + // The identifier names the branch switched to, as Java's `newCatalogEnvironment` rebuilds it, + // so `Name()` and `FullName()` answer for the table served rather than the one left behind. + PAIMON_ASSIGN_OR_RAISE(Identifier branch_identifier, + CatalogUtils::BranchIdentifier(identifier_, target_branch)); return FileStoreTable::Create(store_->GetSpecifiedFileSystem(), root_path_, std::shared_ptr(std::move(branch_schema)), - identifier_, CatalogUuid().value_or(""), target_branch, + branch_identifier, CatalogUuid().value_or(""), target_branch, branch_schema_manager, /*snapshot_manager=*/nullptr, store_->GetFsSchemeMap(), /*dynamic_options=*/{}); } @@ -314,12 +317,6 @@ Result> FileStoreTable::NewWrite( snapshot_manager, io_manager, ignore_previous_files); } -Result> FileStoreWrite::Create(std::unique_ptr ctx) { - PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(ctx, "write")); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, FileStoreTableFactory::Create(*ctx)); - return table->NewWrite(std::move(ctx)); -} - Result> FileStoreTable::NewScan( std::unique_ptr context) const { PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "scan")); @@ -422,12 +419,6 @@ Result> FileStoreTable::NewScan( return batch_scan; } -Result> TableScan::Create(std::unique_ptr context) { - PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "scan")); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, FileStoreTableFactory::Create(*context)); - return table->NewScan(std::move(context)); -} - Result> FileStoreTable::CreateTableRead( const std::shared_ptr& internal_context, const std::shared_ptr& memory_pool, @@ -498,12 +489,6 @@ Result> FileStoreTable::NewRead( std::move(fallback_table_read), arrow_pool); } -Result> TableRead::Create(std::unique_ptr ctx) { - PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(ctx, "read")); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, FileStoreTableFactory::Create(*ctx)); - return table->NewRead(std::move(ctx)); -} - Result> FileStoreTable::NewCommit( std::unique_ptr ctx) const { PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(ctx, "commit")); @@ -605,11 +590,4 @@ Result> FileStoreTable::NewCommit( expire_snapshots, GetSchemaManager(), std::move(scan_supplier)); } -Result> FileStoreCommit::Create( - std::unique_ptr ctx) { - PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(ctx, "commit")); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, FileStoreTableFactory::Create(*ctx)); - return table->NewCommit(std::move(ctx)); -} - } // namespace paimon diff --git a/src/paimon/core/table/source/table_read.cpp b/src/paimon/core/table/source/table_read.cpp index e723c54bb..5688c7650 100644 --- a/src/paimon/core/table/source/table_read.cpp +++ b/src/paimon/core/table/source/table_read.cpp @@ -19,8 +19,18 @@ #include "paimon/table/source/table_read.h" +#include "paimon/catalog/table.h" +#include "paimon/core/table/file_store_table_factory.h" +#include "paimon/core/utils/context_utils.h" + namespace paimon { +Result> TableRead::Create(std::unique_ptr ctx) { + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(ctx, "read")); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, FileStoreTableFactory::Create(*ctx)); + return table->NewRead(std::move(ctx)); +} + Result> TableRead::CreateCountReader( const std::vector>& splits) { (void)splits; diff --git a/src/paimon/core/table/source/table_scan.cpp b/src/paimon/core/table/source/table_scan.cpp index be5d5b955..d3dd794b4 100644 --- a/src/paimon/core/table/source/table_scan.cpp +++ b/src/paimon/core/table/source/table_scan.cpp @@ -19,10 +19,20 @@ #include "paimon/table/source/table_scan.h" +#include "paimon/catalog/table.h" #include "paimon/common/metrics/metrics_impl.h" +#include "paimon/core/table/file_store_table_factory.h" +#include "paimon/core/utils/context_utils.h" +#include "paimon/scan_context.h" namespace paimon { +Result> TableScan::Create(std::unique_ptr context) { + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "scan")); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr
table, FileStoreTableFactory::Create(*context)); + return table->NewScan(std::move(context)); +} + std::shared_ptr TableScan::GetMetrics() const { return std::make_shared(); } diff --git a/src/paimon/core/table/table_test.cpp b/src/paimon/core/table/table_test.cpp index 5100dfd78..8b7b2da45 100644 --- a/src/paimon/core/table/table_test.cpp +++ b/src/paimon/core/table/table_test.cpp @@ -456,6 +456,10 @@ TEST(TableTest, TestSwitchToBranch) { auto dev_file_store_table = std::dynamic_pointer_cast(dev_table); ASSERT_TRUE(dev_file_store_table != nullptr); EXPECT_EQ("dev", dev_file_store_table->CurrentBranch()); + // The identifier names the branch switched to, so `Name()` and `FullName()` answer for the + // table served, and a caller that reloads through `FullName()` lands on the same branch. + EXPECT_EQ("tbl$branch_dev", dev_table->Name()); + EXPECT_EQ("tbl$branch_dev", dev_table->FullName()); ASSERT_EQ((std::vector{"c0"}), dev_table->LatestSchema()->FieldNames()); auto dev_table_schema = std::dynamic_pointer_cast(dev_table->LatestSchema()); ASSERT_TRUE(dev_table_schema != nullptr); @@ -480,6 +484,9 @@ TEST(TableTest, TestSwitchToBranch) { auto main_again_file_store_table = std::dynamic_pointer_cast(main_again); ASSERT_TRUE(main_again_file_store_table != nullptr); EXPECT_EQ("main", main_again_file_store_table->CurrentBranch()); + // Switching back leaves no branch in the name either, the way the main branch is addressed. + EXPECT_EQ("tbl", main_again->Name()); + EXPECT_EQ("tbl", main_again->FullName()); ASSERT_EQ((std::vector{"id"}), main_again->LatestSchema()->FieldNames()); auto main_again_schema = std::dynamic_pointer_cast(main_again->LatestSchema()); ASSERT_TRUE(main_again_schema != nullptr);