From 3c50bebd8b9229322af675ff2da527a8345b4376 Mon Sep 17 00:00:00 2001 From: Gaurav Sharma <223556219+Copilot@users.noreply.github.com> Date: Tue, 15 Sep 2026 13:07:24 +0530 Subject: [PATCH 01/15] FEAT: Add table-valued parameter support Accept materialized nested row sequences for stored-procedure TVP parameters. Resolve the declared table type and child schema from prepared ODBC metadata so empty and typed-NULL rows bind correctly without a public wrapper. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- CHANGELOG.md | 4 + mssql_python/cursor.py | 75 ++- mssql_python/pybind/ddbc_bindings.cpp | 652 ++++++++++++++++++++-- mssql_python/pybind/ddbc_bindings.h | 30 +- mssql_python/pybind/param_detect.hpp | 19 +- tests/test_004_cursor.py | 3 +- tests/test_028_table_valued_parameters.py | 383 +++++++++++++ 7 files changed, 1099 insertions(+), 67 deletions(-) create mode 100644 tests/test_028_table_valued_parameters.py diff --git a/CHANGELOG.md b/CHANGELOG.md index d4f9787e2..e4eecb9a0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ## [Unreleased] ### Added +- **GH-319:** Added input table-valued parameter support for stored procedure + calls. Pass a materialized list or tuple of rows as one `execute()` parameter, + including empty tables and positional or named parameters. The Rust ODBC + provider does not yet support TVPs. - New feature: Support for macOS and Linux. - Documentation: Added API documentation in the Wiki. - New `token_provider=` parameter on `connect()` / `Connection` for Microsoft diff --git a/mssql_python/cursor.py b/mssql_python/cursor.py index 1cb12cb48..8d2d7d56e 100644 --- a/mssql_python/cursor.py +++ b/mssql_python/cursor.py @@ -31,6 +31,7 @@ from mssql_python.row import Row from mssql_python.perf_timer import perf_phase from mssql_python import get_settings +from mssql_python.odbc_provider import ProviderManager, PROVIDER_MSSQL_ODBC from mssql_python.parameter_helper import ( detect_and_convert_parameters, parse_pyformat_params, @@ -365,6 +366,7 @@ def __init__(self, connection: "Connection", timeout: int = 0) -> None: # an HSTMT exists. self.closed: bool = False self.hstmt: Optional[Any] = None + self._tvp_metadata_hstmt: Optional[Any] = None self._connection: "Connection" = connection # Store as private attribute self._timeout: int = timeout @@ -996,7 +998,7 @@ def _allocate_statement_handle(self) -> None: """ self.hstmt = self._connection._conn.alloc_statement_handle() - def _set_timeout(self) -> None: + def _set_timeout(self, statement_handle=None) -> None: """ Set the query timeout attribute on the statement handle. This is called once when the cursor is created and after any handle reallocation. @@ -1007,7 +1009,7 @@ def _set_timeout(self) -> None: try: timeout_value = int(self._timeout) ret = ddbc_bindings.DDBCSQLSetStmtAttr( - self.hstmt, + statement_handle or self.hstmt, ddbc_sql_const.SQL_ATTR_QUERY_TIMEOUT.value, timeout_value, ) @@ -1081,6 +1083,9 @@ def close(self) -> None: self.hstmt.free() self.hstmt = None logger.debug("SQLFreeHandle succeeded") + if self._tvp_metadata_hstmt: + self._tvp_metadata_hstmt.free() + self._tvp_metadata_hstmt = None self._clear_rownumber() self.closed = True @@ -1730,8 +1735,9 @@ def execute( # pylint: disable=too-many-locals,too-many-branches,too-many-state # # Important: If you pass a tuple/list/dict as the ONLY argument, # it will be unwrapped for parameter binding. This means you cannot - # pass a tuple as a single parameter value (but SQL Server doesn't - # support tuple types as parameter values anyway). + # pass a tuple as a single scalar parameter value. A TVP remains one + # parameter by wrapping its rows in the ordinary outer parameter tuple: + # execute("EXEC dbo.proc ?", (rows,)) with perf_phase("py::execute::param_prep"): if parameters: # Check if single parameter is a nested container that should be unwrapped @@ -1769,6 +1775,11 @@ def execute( # pylint: disable=too-many-locals,too-many-branches,too-many-state # Validate that inputsizes matches parameter count if both are present if parameters and self._inputsizes: + if any(isinstance(parameter, (list, tuple)) for parameter in parameters): + raise NotSupportedError( + "setinputsizes does not support table-valued parameters", + "Remove setinputsizes for statements that pass TVP rows", + ) if len(self._inputsizes) != len(parameters): warnings.warn( @@ -1786,22 +1797,33 @@ def execute( # pylint: disable=too-many-locals,too-many-branches,too-many-state self.is_stmt_prepared = [False] effective_use_prepare = use_prepare and not same_sql - with perf_phase("py::execute::cpp_call"): - if parameters: - ret = ddbc_bindings.DDBCSQLExecute( - self.hstmt, - operation, - parameters, - self._inputsizes, - self.is_stmt_prepared, - effective_use_prepare, - encoding_settings, - ) - else: - ret = ddbc_bindings.DDBCSQLExecDirect(self.hstmt, operation) - # Check return code try: - + with perf_phase("py::execute::cpp_call"): + if parameters: + if any(isinstance(parameter, (list, tuple)) for parameter in parameters): + if ProviderManager.effective() == PROVIDER_MSSQL_ODBC: + raise NotSupportedError( + "Table-valued parameters are not supported by " + "the mssql-odbc provider", + "Use the default msodbcsql18 provider for TVP statements", + ) + if self._tvp_metadata_hstmt is None: + self._tvp_metadata_hstmt = ( + self._connection._conn.alloc_statement_handle() + ) + self._set_timeout(self._tvp_metadata_hstmt) + ret = ddbc_bindings.DDBCSQLExecute( + self.hstmt, + self._tvp_metadata_hstmt, + operation, + parameters, + self._inputsizes, + self.is_stmt_prepared, + effective_use_prepare, + encoding_settings, + ) + else: + ret = ddbc_bindings.DDBCSQLExecDirect(self.hstmt, operation) # Check for errors but don't raise exceptions for info/warning messages check_error(ddbc_sql_const.SQL_HANDLE_STMT.value, self.hstmt, ret) except Exception as e: # pylint: disable=broad-exception-caught @@ -2415,7 +2437,6 @@ def executemany( # pylint: disable=too-many-locals,too-many-branches,too-many-s if hasattr(seq_of_parameters, "__getitem__") else next(iter(seq_of_parameters)) ) - if isinstance(first_row, dict): # pyformat style - convert all rows # Parse parameter names from SQL (determines order for all rows) @@ -2480,11 +2501,15 @@ def executemany( # pylint: disable=too-many-locals,too-many-branches,too-many-s # Prepare parameter type information with perf_phase("py::executemany::param_type_detection"): for col_index in range(param_count): - column = ( - [row[col_index] for row in seq_of_parameters] - if hasattr(seq_of_parameters, "__getitem__") - else [] - ) + column = [] + for row in seq_of_parameters: + value = row[col_index] + if isinstance(value, (list, tuple)): + raise NotSupportedError( + "executemany does not support table-valued parameters", + "Use execute for each TVP operation", + ) + column.append(value) sample_value, min_val, max_val, _ = self._compute_column_type(column) if self._inputsizes and col_index < len(self._inputsizes): diff --git a/mssql_python/pybind/ddbc_bindings.cpp b/mssql_python/pybind/ddbc_bindings.cpp index 35a3b6da4..bcc293752 100644 --- a/mssql_python/pybind/ddbc_bindings.cpp +++ b/mssql_python/pybind/ddbc_bindings.cpp @@ -23,6 +23,10 @@ #include // std::forward #include // CPython datetime API (PyDateTime_IMPORT, PyDateTime_GET_*, etc.) +SQLRETURN BindParameterArray(SqlHandle& handle, SQLHANDLE hStmt, const py::list& columnwise_params, + std::vector& paramInfos, size_t paramSetSize, + std::vector>& paramBuffers, + const std::string& charEncoding, bool resolveUnknownTypes = true); //------------------------------------------------------------------------------------------------- // Macro definitions @@ -227,6 +231,7 @@ SQLExecuteFunc SQLExecute_ptr = nullptr; SQLRowCountFunc SQLRowCount_ptr = nullptr; SQLGetStmtAttrFunc SQLGetStmtAttr_ptr = nullptr; SQLSetDescFieldFunc SQLSetDescField_ptr = nullptr; +SQLGetDescFieldFunc SQLGetDescField_ptr = nullptr; // Data retrieval APIs SQLFetchFunc SQLFetch_ptr = nullptr; @@ -448,6 +453,559 @@ static void PreResolveUnknownNullTypes(SqlHandle& handle, SQLHANDLE hStmt, } } +static bool IsIntegerCType(SQLSMALLINT cType) { + return cType == SQL_C_TINYINT || cType == SQL_C_UTINYINT || cType == SQL_C_SHORT || + cType == SQL_C_LONG || cType == SQL_C_SBIGINT; +} + +static bool IsTextSqlType(SQLSMALLINT sqlType) { + return sqlType == SQL_VARCHAR || sqlType == SQL_WVARCHAR; +} + +static ParamInfo MergeTableColumnInfo(const py::list& values, + const std::vector& valueInfos, + size_t columnIndex) { + ParamInfo merged; + bool hasValue = false; + bool integerColumn = false; + bool textColumn = false; + bool hasInteger = false; + int64_t minInteger = 0; + int64_t maxInteger = 0; + SQLULEN maxSize = 0; + SQLULEN maxIntegerDigits = 0; + SQLSMALLINT maxScale = 0; + + for (size_t rowIndex = 0; rowIndex < valueInfos.size(); ++rowIndex) { + const ParamInfo& current = valueInfos[rowIndex]; + if (values[rowIndex].is_none()) { + continue; + } + if (current.isTVP) { + throw py::type_error("TVP cells cannot contain nested row sequences"); + } + + const bool currentInteger = IsIntegerCType(current.paramCType); + const bool currentText = + current.paramCType == SQL_C_WCHAR && IsTextSqlType(current.paramSQLType); + if (!hasValue) { + merged = current; + integerColumn = currentInteger; + textColumn = currentText; + hasValue = true; + } else if (integerColumn && currentInteger) { + // Integer widths are merged below from the full observed range. + } else if (textColumn && currentText) { + if (current.paramSQLType == SQL_WVARCHAR) { + merged.paramSQLType = SQL_WVARCHAR; + } + } else if (merged.paramCType != current.paramCType || + merged.paramSQLType != current.paramSQLType) { + throw py::type_error("TVP column " + std::to_string(columnIndex) + + " contains incompatible Python types at row " + + std::to_string(rowIndex)); + } + + if (currentInteger) { + int overflow = 0; + int64_t value = PyLong_AsLongLongAndOverflow(values[rowIndex].ptr(), &overflow); + if (overflow != 0) { + PyErr_Clear(); + throw py::value_error("TVP integer cell is out of range for SQL BIGINT"); + } + if (PyErr_Occurred()) + throw py::error_already_set(); + if (!integerColumn) { + throw py::type_error("TVP column " + std::to_string(columnIndex) + + " contains incompatible Python types at row " + + std::to_string(rowIndex)); + } + if (!hasInteger) { + minInteger = value; + maxInteger = value; + hasInteger = true; + } else { + minInteger = std::min(minInteger, value); + maxInteger = std::max(maxInteger, value); + } + } else if (currentText) { + SQLULEN size = + current.isDAE ? static_cast(current.utf16Len) : current.columnSize; + maxSize = std::max(maxSize, size); + } else if (current.paramCType == SQL_C_BINARY) { + Py_ssize_t size = PyObject_Length(values[rowIndex].ptr()); + if (size < 0) + throw py::error_already_set(); + maxSize = std::max(maxSize, static_cast(size)); + } else if (current.paramCType == SQL_C_NUMERIC) { + SQLULEN integerDigits = + current.columnSize > static_cast(current.decimalDigits) + ? current.columnSize - static_cast(current.decimalDigits) + : 0; + maxIntegerDigits = std::max(maxIntegerDigits, integerDigits); + maxScale = std::max(maxScale, current.decimalDigits); + } else { + maxSize = std::max(maxSize, current.columnSize); + maxScale = std::max(maxScale, current.decimalDigits); + } + } + + if (!hasValue) { + merged.paramCType = SQL_C_DEFAULT; + merged.paramSQLType = SQL_VARCHAR; + merged.columnSize = 1; + merged.decimalDigits = 0; + return merged; + } + + if (integerColumn) { + if (minInteger >= 0 && maxInteger <= UINT8_MAX) { + merged.paramCType = SQL_C_UTINYINT; + merged.paramSQLType = SQL_TINYINT; + merged.columnSize = 3; + } else if (minInteger >= INT16_MIN && maxInteger <= INT16_MAX) { + merged.paramCType = SQL_C_SHORT; + merged.paramSQLType = SQL_SMALLINT; + merged.columnSize = 5; + } else if (minInteger >= INT32_MIN && maxInteger <= INT32_MAX) { + merged.paramCType = SQL_C_LONG; + merged.paramSQLType = SQL_INTEGER; + merged.columnSize = 10; + } else { + merged.paramCType = SQL_C_SBIGINT; + merged.paramSQLType = SQL_BIGINT; + merged.columnSize = 19; + } + merged.decimalDigits = 0; + } else if (textColumn) { + merged.columnSize = std::max(maxSize, 1); + if (merged.paramSQLType == SQL_WVARCHAR && maxSize > MAX_INLINE_CHAR) { + merged.paramSQLType = SQL_WLONGVARCHAR; + } else if (merged.paramSQLType == SQL_VARCHAR && maxSize > MAX_INLINE_BINARY) { + merged.paramSQLType = SQL_LONGVARCHAR; + } + merged.isDAE = false; + } else if (merged.paramCType == SQL_C_BINARY) { + merged.columnSize = std::max(maxSize, 1); + if (maxSize > MAX_INLINE_BINARY) { + merged.paramSQLType = SQL_LONGVARBINARY; + } + merged.isDAE = false; + } else if (merged.paramCType == SQL_C_NUMERIC) { + merged.decimalDigits = maxScale; + merged.columnSize = std::max(maxIntegerDigits + maxScale, 1); + } else { + merged.columnSize = std::max(maxSize, 1); + merged.decimalDigits = maxScale; + } + + return merged; +} + +static bool NormalizeTableDecimalColumn(py::list& values, ParamInfo& info, size_t columnIndex) { + bool hasDecimal = false; + bool hasOtherValue = false; + PyObject* decimalType = PyTypeCache::get_decimal_class(); + for (const py::handle value : values) { + if (value.is_none()) { + continue; + } + int isDecimal = PyObject_IsInstance(value.ptr(), decimalType); + if (isDecimal == -1) + throw py::error_already_set(); + hasDecimal = hasDecimal || isDecimal == 1; + hasOtherValue = hasOtherValue || isDecimal == 0; + } + if (!hasDecimal) { + return false; + } + if (hasOtherValue) { + throw py::type_error("TVP column " + std::to_string(columnIndex) + + " contains incompatible Python types"); + } + + py::str formatSpec("f"); + SQLULEN maxSize = 1; + for (size_t rowIndex = 0; rowIndex < values.size(); ++rowIndex) { + if (values[rowIndex].is_none()) { + continue; + } + py::object isFinite = + steal(PyObject_CallMethod(values[rowIndex].ptr(), "is_finite", nullptr)); + if (!isFinite) + throw py::error_already_set(); + int finite = PyObject_IsTrue(isFinite.ptr()); + if (finite == -1) + throw py::error_already_set(); + if (finite == 0) { + throw py::value_error("Cannot bind non-finite Decimal (NaN/Infinity) in a TVP"); + } + py::object formatted = steal(PyObject_Format(values[rowIndex].ptr(), formatSpec.ptr())); + if (!formatted) + throw py::error_already_set(); + maxSize = std::max(maxSize, static_cast(PyUnicode_GET_LENGTH(formatted.ptr()))); + values[rowIndex] = std::move(formatted); + } + + info.paramCType = SQL_C_CHAR; + info.paramSQLType = SQL_NUMERIC; + info.columnSize = maxSize; + info.bufferSize = maxSize; + info.decimalDigits = 0; + return true; +} + +static std::vector +DetectTableColumnTypes(py::list& columnValues, const std::vector& declaredColumns) { + std::vector columnInfos(columnValues.size()); + for (size_t columnIndex = 0; columnIndex < columnValues.size(); ++columnIndex) { + py::list values = columnValues[columnIndex].cast(); + if ((declaredColumns[columnIndex].sqlType == SQL_DECIMAL || + declaredColumns[columnIndex].sqlType == SQL_NUMERIC) && + NormalizeTableDecimalColumn(values, columnInfos[columnIndex], columnIndex)) { + continue; + } + std::vector valueInfos = DetectParamTypes(values.ptr(), Py_None); + columnInfos[columnIndex] = MergeTableColumnInfo(values, valueInfos, columnIndex); + } + return columnInfos; +} + +static SQLRETURN ReadDescriptorString(SQLHDESC descriptor, SQLSMALLINT recordNumber, + SQLSMALLINT fieldIdentifier, std::u16string& value) { + SQLWCHAR buffer[256] = {}; + SQLINTEGER length = 0; + SQLRETURN rc = SQLGetDescField_ptr(descriptor, recordNumber, fieldIdentifier, buffer, + static_cast(sizeof(buffer)), &length); + if (!SQL_SUCCEEDED(rc)) { + return rc; + } + if (length <= 0) { + value.clear(); + return rc; + } + size_t codeUnits = std::min(static_cast(length) / sizeof(SQLWCHAR), + (sizeof(buffer) / sizeof(buffer[0])) - 1); + value = dupeSqlWCharAsUtf16Le(buffer, codeUnits); + return rc; +} + +[[noreturn]] static void ThrowTableMetadataError(const SqlHandlePtr& metadataHandle, SQLRETURN rc, + const std::string& operation) { + ErrorInfo error = SQLCheckError_Wrap(SQL_HANDLE_STMT, metadataHandle, rc); + py::object raiseException = + py::module_::import("mssql_python.exceptions").attr("raise_exception"); + raiseException(error.sqlState, operation + ": " + error.ddbcErrorMsg); + throw std::runtime_error(operation); +} + +class TableMetadataScope { + public: + explicit TableMetadataScope(SQLHANDLE statementHandle) : handle(statementHandle) {} + ~TableMetadataScope() { + SQLFreeStmt_ptr(handle, SQL_CLOSE); + SQLSetStmtAttr_ptr( + handle, SQL_SOPT_SS_NAME_SCOPE, + reinterpret_cast(static_cast(SQL_SS_NAME_SCOPE_TABLE)), + SQL_IS_INTEGER); + SQLSetStmtAttr_ptr(handle, SQL_ATTR_METADATA_ID, + reinterpret_cast(static_cast(SQL_FALSE)), + SQL_IS_INTEGER); + } + + private: + SQLHANDLE handle; +}; + +static std::vector LoadTableColumnMetadata(const SqlHandlePtr& metadataHandle, + const std::u16string& catalog, + const std::u16string& schema, + const std::u16string& typeName) { + if (!metadataHandle || !metadataHandle->get()) { + throw std::runtime_error("TVP column metadata requires a valid statement handle"); + } + + SQLHANDLE hMetadataStmt = metadataHandle->get(); + SQLFreeStmt_ptr(hMetadataStmt, SQL_CLOSE); + SQLFreeStmt_ptr(hMetadataStmt, SQL_RESET_PARAMS); + + SQLRETURN rc = SQLSetStmtAttr_ptr(hMetadataStmt, SQL_ATTR_METADATA_ID, + reinterpret_cast(static_cast(SQL_TRUE)), + SQL_IS_INTEGER); + if (!SQL_SUCCEEDED(rc)) { + ThrowTableMetadataError(metadataHandle, rc, "Failed to enable metadata identifier mode"); + } + TableMetadataScope metadataScope(hMetadataStmt); + + rc = SQLSetStmtAttr_ptr( + hMetadataStmt, SQL_SOPT_SS_NAME_SCOPE, + reinterpret_cast(static_cast(SQL_SS_NAME_SCOPE_TABLE_TYPE)), + SQL_IS_INTEGER); + if (!SQL_SUCCEEDED(rc)) { + ThrowTableMetadataError(metadataHandle, rc, "Failed to select table-type metadata scope"); + } + { + py::gil_scoped_release release; + rc = SQLColumns_ptr(hMetadataStmt, + catalog.empty() ? nullptr : reinterpretU16stringAsSqlWChar(catalog), + catalog.empty() ? 0 : SQL_NTS, + schema.empty() ? nullptr : reinterpretU16stringAsSqlWChar(schema), + schema.empty() ? 0 : SQL_NTS, reinterpretU16stringAsSqlWChar(typeName), + SQL_NTS, nullptr, 0); + } + if (!SQL_SUCCEEDED(rc)) { + ThrowTableMetadataError(metadataHandle, rc, "Failed to discover TVP columns"); + } + + std::vector> orderedColumns; + while (true) { + { + py::gil_scoped_release release; + rc = SQLFetch_ptr(hMetadataStmt); + } + if (rc == SQL_NO_DATA) { + break; + } + if (!SQL_SUCCEEDED(rc)) { + ThrowTableMetadataError(metadataHandle, rc, "Failed to fetch TVP column metadata"); + } + + SQLSMALLINT sqlType = SQL_UNKNOWN_TYPE; + SQLINTEGER columnSize = 0; + SQLSMALLINT decimalDigits = 0; + SQLINTEGER ordinal = 0; + SQLLEN indicator = 0; + + rc = SQLGetData_ptr(hMetadataStmt, 5, SQL_C_SSHORT, &sqlType, sizeof(sqlType), &indicator); + if (SQL_SUCCEEDED(rc)) { + rc = SQLGetData_ptr(hMetadataStmt, 7, SQL_C_SLONG, &columnSize, sizeof(columnSize), + &indicator); + } + if (SQL_SUCCEEDED(rc)) { + rc = SQLGetData_ptr(hMetadataStmt, 9, SQL_C_SSHORT, &decimalDigits, + sizeof(decimalDigits), &indicator); + if (indicator == SQL_NULL_DATA) { + decimalDigits = 0; + rc = SQL_SUCCESS; + } + } + if (SQL_SUCCEEDED(rc)) { + rc = SQLGetData_ptr(hMetadataStmt, 17, SQL_C_SLONG, &ordinal, sizeof(ordinal), + &indicator); + } + if (!SQL_SUCCEEDED(rc)) { + ThrowTableMetadataError(metadataHandle, rc, "Failed to read TVP column metadata"); + } + + orderedColumns.push_back({ + ordinal, + {sqlType, static_cast(std::max(columnSize, 0)), decimalDigits}, + }); + } + + if (orderedColumns.empty()) { + throw std::runtime_error("No column metadata was returned for the table type"); + } + std::sort(orderedColumns.begin(), orderedColumns.end(), + [](const auto& left, const auto& right) { return left.first < right.first; }); + + std::vector columns; + columns.reserve(orderedColumns.size()); + for (const auto& entry : orderedColumns) { + columns.push_back(entry.second); + } + return columns; +} + +static SQLRETURN ResolveTableValuedParamTypes(SqlHandle& handle, SQLHANDLE hStmt, + const SqlHandlePtr& metadataHandle, + std::vector& paramInfos) { + for (size_t paramIndex = 0; paramIndex < paramInfos.size(); ++paramIndex) { + ParamInfo& info = paramInfos[paramIndex]; + if (!info.isTVP) { + continue; + } + + if (handle.tvpCache.find(static_cast(paramIndex)) == handle.tvpCache.end()) { + DescribedParamInfo described; + SQLSMALLINT nullable; + RETCODE rc; + { + py::gil_scoped_release release; + rc = SQLDescribeParam_ptr(hStmt, static_cast(paramIndex + 1), + &described.sqlType, &described.columnSize, + &described.decimalDigits, &nullable); + } + if (!SQL_SUCCEEDED(rc)) { + return rc; + } + if (described.sqlType != SQL_SS_TABLE) { + throw py::type_error( + "Nested row sequences can only be bound to a table-valued parameter"); + } + + SQLHDESC implementationDescriptor = nullptr; + rc = SQLGetStmtAttr_ptr(hStmt, SQL_ATTR_IMP_PARAM_DESC, &implementationDescriptor, 0, + nullptr); + if (!SQL_SUCCEEDED(rc)) { + return rc; + } + + std::u16string catalog; + std::u16string schema; + std::u16string typeName; + const SQLSMALLINT recordNumber = static_cast(paramIndex + 1); + rc = ReadDescriptorString(implementationDescriptor, recordNumber, + SQL_CA_SS_CATALOG_NAME, catalog); + if (SQL_SUCCEEDED(rc)) { + rc = ReadDescriptorString(implementationDescriptor, recordNumber, + SQL_CA_SS_SCHEMA_NAME, schema); + } + if (SQL_SUCCEEDED(rc)) { + rc = ReadDescriptorString(implementationDescriptor, recordNumber, + SQL_CA_SS_TYPE_NAME, typeName); + } + if (!SQL_SUCCEEDED(rc)) { + return rc; + } + if (typeName.empty()) { + throw std::runtime_error("The prepared statement did not expose a TVP type name"); + } + handle.tvpCache[static_cast(paramIndex)] = { + catalog, + schema, + typeName, + LoadTableColumnMetadata(metadataHandle, catalog, schema, typeName), + }; + } + info.paramSQLType = SQL_SS_TABLE; + info.paramCType = SQL_C_BINARY; + info.decimalDigits = 0; + } + return SQL_SUCCESS; +} + +static SQLRETURN BindTableValuedParameter(SqlHandle& handle, SQLHANDLE hStmt, int paramIndex, + const py::handle& param, + std::vector>& paramBuffers, + const std::string& charEncoding) { + const py::sequence rows = py::reinterpret_borrow(param); + const SQLSMALLINT recordNumber = static_cast(paramIndex + 1); + + size_t columnCount = 0; + py::list columnValues; + std::vector columnInfos; + auto metadataEntry = handle.tvpCache.find(paramIndex); + if (metadataEntry == handle.tvpCache.end()) { + throw std::runtime_error("TVP column metadata was not resolved before binding"); + } + const TvpParamInfo& tableMetadata = metadataEntry->second; + const std::vector& declaredColumns = tableMetadata.columns; + if (!rows.empty()) { + const py::handle firstRow = rows[0]; + if (!PyList_Check(firstRow.ptr()) && !PyTuple_Check(firstRow.ptr())) { + throw py::type_error("TVP rows must be list or tuple objects"); + } + columnCount = py::len(firstRow); + if (columnCount == 0) { + throw py::value_error("TVP rows must contain at least one column"); + } + if (columnCount != declaredColumns.size()) { + throw py::value_error("TVP rows contain " + std::to_string(columnCount) + + " columns; the declared table type requires " + + std::to_string(declaredColumns.size())); + } + columnValues = py::list(columnCount); + for (size_t columnIndex = 0; columnIndex < columnCount; ++columnIndex) { + columnValues[columnIndex] = py::list(); + } + for (size_t rowIndex = 0; rowIndex < rows.size(); ++rowIndex) { + const py::sequence row = py::reinterpret_borrow(rows[rowIndex]); + if (!PyList_Check(row.ptr()) && !PyTuple_Check(row.ptr())) { + throw py::type_error("TVP rows must be list or tuple objects"); + } + if (static_cast(py::len(row)) != columnCount) { + throw py::value_error("TVP rows must all contain the same number of columns"); + } + for (size_t columnIndex = 0; columnIndex < columnCount; ++columnIndex) { + columnValues[columnIndex].cast().append(row[columnIndex]); + } + } + columnInfos = DetectTableColumnTypes(columnValues, declaredColumns); + for (size_t columnIndex = 0; columnIndex < columnCount; ++columnIndex) { + ParamInfo& inferred = columnInfos[columnIndex]; + const TvpColumnInfo& declared = declaredColumns[columnIndex]; + inferred.paramSQLType = declared.sqlType; + if (declared.columnSize == 0) { + if (declared.sqlType == SQL_WVARCHAR && inferred.columnSize > MAX_INLINE_CHAR) { + inferred.paramSQLType = SQL_WLONGVARCHAR; + } else if (declared.sqlType == SQL_VARCHAR && + inferred.columnSize > MAX_INLINE_BINARY) { + inferred.paramSQLType = SQL_LONGVARCHAR; + } else if (declared.sqlType == SQL_VARBINARY && + inferred.columnSize > MAX_INLINE_BINARY) { + inferred.paramSQLType = SQL_LONGVARBINARY; + } + } + inferred.decimalDigits = declared.decimalDigits; + if (inferred.paramCType == SQL_C_DEFAULT || declared.sqlType == SQL_DECIMAL || + declared.sqlType == SQL_NUMERIC) { + inferred.columnSize = std::max(declared.columnSize, 1); + } + } + } + + auto* typeName = AllocateParamBuffer(paramBuffers, tableMetadata.typeName); + auto* schema = AllocateParamBuffer(paramBuffers, tableMetadata.schema); + auto* rowCount = AllocateParamBuffer(paramBuffers); + // TVPs overload ColumnSize with row capacity and the indicator with the rows available. + *rowCount = rows.empty() ? SQL_DEFAULT_PARAM : static_cast(rows.size()); + + SQLRETURN rc = SQLBindParameter_ptr( + hStmt, static_cast(recordNumber), SQL_PARAM_INPUT, SQL_C_BINARY, SQL_SS_TABLE, + static_cast(rows.size()), 0, typeName->data(), + static_cast(typeName->size() * sizeof(SQLWCHAR)), rowCount); + if (!SQL_SUCCEEDED(rc)) { + return rc; + } + if (!schema->empty()) { + SQLHDESC implementationDescriptor = nullptr; + rc = SQLGetStmtAttr_ptr(hStmt, SQL_ATTR_IMP_PARAM_DESC, &implementationDescriptor, 0, + nullptr); + if (!SQL_SUCCEEDED(rc)) { + return rc; + } + rc = SQLSetDescField_ptr(implementationDescriptor, recordNumber, SQL_CA_SS_SCHEMA_NAME, + reinterpretU16stringAsSqlWChar(*schema), + static_cast(schema->size() * sizeof(SQLWCHAR))); + if (!SQL_SUCCEEDED(rc)) { + return rc; + } + } + if (rows.empty()) { + return rc; + } + + rc = SQLSetStmtAttr_ptr(hStmt, SQL_SOPT_SS_PARAM_FOCUS, + reinterpret_cast(static_cast(recordNumber)), + SQL_IS_INTEGER); + if (!SQL_SUCCEEDED(rc)) { + return rc; + } + + SQLRETURN bindRc = SQL_ERROR; + try { + bindRc = BindParameterArray(handle, hStmt, columnValues, columnInfos, rows.size(), + paramBuffers, charEncoding, false); + } catch (...) { + SQLSetStmtAttr_ptr(hStmt, SQL_SOPT_SS_PARAM_FOCUS, nullptr, SQL_IS_INTEGER); + throw; + } + const SQLRETURN restoreRc = + SQLSetStmtAttr_ptr(hStmt, SQL_SOPT_SS_PARAM_FOCUS, nullptr, SQL_IS_INTEGER); + return SQL_SUCCEEDED(bindRc) ? restoreRc : bindRc; +} + // Given a list of parameters and their ParamInfo, calls SQLBindParameter on // each of them with appropriate arguments SQLRETURN BindParameters(SqlHandle& handle, SQLHANDLE hStmt, const py::list& params, @@ -473,6 +1031,15 @@ SQLRETURN BindParameters(SqlHandle& handle, SQLHANDLE hStmt, const py::list& par SQLLEN bufferLength = 0; SQLLEN* strLenOrIndPtr = nullptr; + if (paramInfo.isTVP) { + RETCODE rc = BindTableValuedParameter(handle, hStmt, paramIndex, param, paramBuffers, + charEncoding); + if (!SQL_SUCCEEDED(rc)) { + return rc; + } + continue; + } + // TODO: Add more data types like money, guid, interval, TVPs etc. switch (paramInfo.paramCType) { case SQL_C_CHAR: { @@ -1445,6 +2012,7 @@ DriverHandle LoadDriverOrThrowException() { SQLRowCount_ptr = GetFunctionPointer(handle, "SQLRowCount"); SQLGetStmtAttr_ptr = GetFunctionPointer(handle, "SQLGetStmtAttrW"); SQLSetDescField_ptr = GetFunctionPointer(handle, "SQLSetDescFieldW"); + SQLGetDescField_ptr = GetFunctionPointer(handle, "SQLGetDescFieldW"); SQLFetch_ptr = GetFunctionPointer(handle, "SQLFetch"); SQLFetchScroll_ptr = GetFunctionPointer(handle, "SQLFetchScroll"); @@ -1477,17 +2045,17 @@ DriverHandle LoadDriverOrThrowException() { SQLDescribeParam_ptr = GetFunctionPointer(handle, "SQLDescribeParam"); - bool success = SQLAllocHandle_ptr && SQLSetEnvAttr_ptr && SQLSetConnectAttr_ptr && - SQLSetStmtAttr_ptr && SQLGetConnectAttr_ptr && SQLDriverConnect_ptr && - SQLExecDirect_ptr && SQLPrepare_ptr && SQLBindParameter_ptr && SQLExecute_ptr && - SQLRowCount_ptr && SQLGetStmtAttr_ptr && SQLSetDescField_ptr && SQLFetch_ptr && - SQLFetchScroll_ptr && SQLGetData_ptr && SQLNumResultCols_ptr && SQLBindCol_ptr && - SQLDescribeCol_ptr && SQLMoreResults_ptr && SQLColAttribute_ptr && - SQLEndTran_ptr && SQLDisconnect_ptr && SQLFreeHandle_ptr && SQLFreeStmt_ptr && - SQLGetDiagRec_ptr && SQLGetInfo_ptr && SQLParamData_ptr && SQLPutData_ptr && - SQLTables_ptr && SQLDescribeParam_ptr && SQLGetTypeInfo_ptr && - SQLProcedures_ptr && SQLForeignKeys_ptr && SQLPrimaryKeys_ptr && - SQLSpecialColumns_ptr && SQLStatistics_ptr && SQLColumns_ptr; + bool success = + SQLAllocHandle_ptr && SQLSetEnvAttr_ptr && SQLSetConnectAttr_ptr && SQLSetStmtAttr_ptr && + SQLGetConnectAttr_ptr && SQLDriverConnect_ptr && SQLExecDirect_ptr && SQLPrepare_ptr && + SQLBindParameter_ptr && SQLExecute_ptr && SQLRowCount_ptr && SQLGetStmtAttr_ptr && + SQLSetDescField_ptr && SQLGetDescField_ptr && SQLFetch_ptr && SQLFetchScroll_ptr && + SQLGetData_ptr && SQLNumResultCols_ptr && SQLBindCol_ptr && SQLDescribeCol_ptr && + SQLMoreResults_ptr && SQLColAttribute_ptr && SQLEndTran_ptr && SQLDisconnect_ptr && + SQLFreeHandle_ptr && SQLFreeStmt_ptr && SQLGetDiagRec_ptr && SQLGetInfo_ptr && + SQLParamData_ptr && SQLPutData_ptr && SQLTables_ptr && SQLDescribeParam_ptr && + SQLGetTypeInfo_ptr && SQLProcedures_ptr && SQLForeignKeys_ptr && SQLPrimaryKeys_ptr && + SQLSpecialColumns_ptr && SQLStatistics_ptr && SQLColumns_ptr; if (!success) { ThrowStdException("Failed to load required function pointers from driver."); @@ -1583,7 +2151,7 @@ void SqlHandle::free() { PERF_TIMER("SqlHandle::free"); if (_handle && SQLFreeHandle_ptr) { // GH-610: Clear describe cache to prevent memory leak. - describeCache.clear(); + clearDescribeCache(); // Check if Python is shutting down using centralized helper function bool pythonShuttingDown = is_python_finalizing(); @@ -2032,12 +2600,9 @@ SQLRETURN SQLTables_wrap(SqlHandlePtr StatementHandle, const std::u16string& cat // When false and not prepared, throws (matching slow path behavior). // --------------------------------------------------------------------------- SQLRETURN SQLExecute_wrap(const SqlHandlePtr statementHandle, - const std::u16string& query, - py::list params, - const py::object& input_sizes, - py::list is_stmt_prepared, - bool use_prepare, - const py::dict& encoding_settings) { + const SqlHandlePtr metadataStatementHandle, const std::u16string& query, + py::list params, const py::object& input_sizes, py::list is_stmt_prepared, + bool use_prepare, const py::dict& encoding_settings) { PERF_TIMER("SQLExecute_wrap"); if (!statementHandle || !statementHandle->get()) { return SQL_INVALID_HANDLE; @@ -2101,6 +2666,10 @@ SQLRETURN SQLExecute_wrap(const SqlHandlePtr statementHandle, } } + rc = ResolveTableValuedParamTypes(*statementHandle, hStmt, metadataStatementHandle, paramInfos); + if (!SQL_SUCCEEDED(rc)) + return rc; + std::vector> paramBuffers; rc = BindParameters(*statementHandle, hStmt, params, paramInfos, paramBuffers, charEncoding); if (!SQL_SUCCEEDED(rc)) return rc; @@ -2206,7 +2775,7 @@ SQLRETURN SQLExecute_wrap(const SqlHandlePtr statementHandle, SQLRETURN BindParameterArray(SqlHandle& handle, SQLHANDLE hStmt, const py::list& columnwise_params, std::vector& paramInfos, size_t paramSetSize, std::vector>& paramBuffers, - const std::string& charEncoding = "utf-8") { + const std::string& charEncoding, bool resolveUnknownTypes) { PERF_TIMER("BindParameterArray"); LOG("BindParameterArray: Starting column-wise array binding - " "param_count=%zu, param_set_size=%zu", @@ -2216,7 +2785,9 @@ SQLRETURN BindParameterArray(SqlHandle& handle, SQLHANDLE hStmt, const py::list& try { // GH-627: resolve unknown NULL array param SQL types before binding any param. - PreResolveUnknownNullTypes(handle, hStmt, paramInfos); + if (resolveUnknownTypes) { + PreResolveUnknownNullTypes(handle, hStmt, paramInfos); + } for (int paramIndex = 0; paramIndex < columnwise_params.size(); ++paramIndex) { const py::list& columnValues = columnwise_params[paramIndex].cast(); ParamInfo& info = paramInfos[paramIndex]; @@ -2275,36 +2846,39 @@ SQLRETURN BindParameterArray(SqlHandle& handle, SQLHANDLE hStmt, const py::list& break; } case SQL_C_WCHAR: { + const SQLULEN valueBufferSize = + info.bufferSize > 0 ? info.bufferSize : info.columnSize; LOG("BindParameterArray: Binding SQL_C_WCHAR array - " "param_index=%d, count=%zu, column_size=%zu", - paramIndex, paramSetSize, info.columnSize); + paramIndex, paramSetSize, valueBufferSize); SQLWCHAR* wcharArray = AllocateParamBufferArray( - tempBuffers, paramSetSize * (info.columnSize + 1)); + tempBuffers, paramSetSize * (valueBufferSize + 1)); strLenOrIndArray = AllocateParamBufferArray(tempBuffers, paramSetSize); for (size_t i = 0; i < paramSetSize; ++i) { if (columnValues[i].is_none()) { strLenOrIndArray[i] = SQL_NULL_DATA; - std::memset(wcharArray + i * (info.columnSize + 1), 0, - (info.columnSize + 1) * sizeof(SQLWCHAR)); + std::memset(wcharArray + i * (valueBufferSize + 1), 0, + (valueBufferSize + 1) * sizeof(SQLWCHAR)); } else { std::u16string wstr = columnValues[i].cast(); // u16string is already UTF-16, so the // original check is sufficient - if (wstr.length() > info.columnSize) { + if (wstr.length() > valueBufferSize) { ThrowStdException("Input string exceeds allowed column size " "at parameter index " + std::to_string(paramIndex)); } - std::memcpy(wcharArray + i * (info.columnSize + 1), wstr.c_str(), + std::memcpy(wcharArray + i * (valueBufferSize + 1), wstr.c_str(), (wstr.length() + 1) * sizeof(SQLWCHAR)); - strLenOrIndArray[i] = SQL_NTS; + strLenOrIndArray[i] = + static_cast(wstr.length() * sizeof(SQLWCHAR)); } } LOG("BindParameterArray: SQL_C_WCHAR bound - " "param_index=%d", paramIndex); dataPtr = wcharArray; - bufferLength = (info.columnSize + 1) * sizeof(SQLWCHAR); + bufferLength = (valueBufferSize + 1) * sizeof(SQLWCHAR); break; } case SQL_C_TINYINT: @@ -2372,17 +2946,19 @@ SQLRETURN BindParameterArray(SqlHandle& handle, SQLHANDLE hStmt, const py::list& } case SQL_C_CHAR: case SQL_C_BINARY: { + const SQLULEN valueBufferSize = + info.bufferSize > 0 ? info.bufferSize : info.columnSize; LOG("BindParameterArray: Binding SQL_C_CHAR/BINARY array - " "param_index=%d, count=%zu, column_size=%zu, encoding='%s'", - paramIndex, paramSetSize, info.columnSize, charEncoding.c_str()); + paramIndex, paramSetSize, valueBufferSize, charEncoding.c_str()); char* charArray = AllocateParamBufferArray( - tempBuffers, paramSetSize * (info.columnSize + 1)); + tempBuffers, paramSetSize * (valueBufferSize + 1)); strLenOrIndArray = AllocateParamBufferArray(tempBuffers, paramSetSize); for (size_t i = 0; i < paramSetSize; ++i) { if (columnValues[i].is_none()) { strLenOrIndArray[i] = SQL_NULL_DATA; - std::memset(charArray + i * (info.columnSize + 1), 0, - info.columnSize + 1); + std::memset(charArray + i * (valueBufferSize + 1), 0, + valueBufferSize + 1); } else { std::string encodedStr; @@ -2412,15 +2988,15 @@ SQLRETURN BindParameterArray(SqlHandle& handle, SQLHANDLE hStmt, const py::list& encodedStr = columnValues[i].cast(); } - if (encodedStr.size() > info.columnSize) { + if (encodedStr.size() > valueBufferSize) { LOG("BindParameterArray: String/binary too " "long - param_index=%d, row=%zu, size=%zu, " "max=%zu", - paramIndex, i, encodedStr.size(), info.columnSize); + paramIndex, i, encodedStr.size(), valueBufferSize); ThrowStdException("Input exceeds column size at index " + std::to_string(i)); } - std::memcpy(charArray + i * (info.columnSize + 1), encodedStr.c_str(), + std::memcpy(charArray + i * (valueBufferSize + 1), encodedStr.c_str(), encodedStr.size()); strLenOrIndArray[i] = static_cast(encodedStr.size()); } @@ -2429,7 +3005,7 @@ SQLRETURN BindParameterArray(SqlHandle& handle, SQLHANDLE hStmt, const py::list& "param_index=%d", paramIndex); dataPtr = charArray; - bufferLength = info.columnSize + 1; + bufferLength = valueBufferSize + 1; break; } case SQL_C_BIT: { @@ -6123,8 +6699,8 @@ PYBIND11_MODULE(ddbc_bindings, m) { }, "Disable global connection pooling and close all pools"); m.def("DDBCSQLExecDirect", &SQLExecDirect_wrap, "Execute a SQL query directly"); m.def("DDBCSQLExecute", &SQLExecute_wrap, - "DetectParamTypes + BindParameters + SQLExecute all in C++", - py::arg("statementHandle"), py::arg("query"), py::arg("params"), + "DetectParamTypes + BindParameters + SQLExecute all in C++", py::arg("statementHandle"), + py::arg("metadataStatementHandle"), py::arg("query"), py::arg("params"), py::arg("inputSizes"), py::arg("isStmtPrepared"), py::arg("usePrepare"), py::arg("encodingSettings")); m.def("SQLExecuteMany", &SQLExecuteMany_wrap, "Execute statement with multiple parameter sets", diff --git a/mssql_python/pybind/ddbc_bindings.h b/mssql_python/pybind/ddbc_bindings.h index 00f04aa09..3558db178 100644 --- a/mssql_python/pybind/ddbc_bindings.h +++ b/mssql_python/pybind/ddbc_bindings.h @@ -50,7 +50,15 @@ using py::literals::operator""_a; #define SQL_SS_XML (-152) #define SQL_SS_UDT (-151) #define SQL_SS_VARIANT (-150) +#define SQL_SS_TABLE (-153) #define SQL_CA_SS_VARIANT_TYPE (1215) +#define SQL_SOPT_SS_PARAM_FOCUS (1236) +#define SQL_SOPT_SS_NAME_SCOPE (1237) +#define SQL_SS_NAME_SCOPE_TABLE (0L) +#define SQL_SS_NAME_SCOPE_TABLE_TYPE (1L) +#define SQL_CA_SS_CATALOG_NAME (1225) +#define SQL_CA_SS_SCHEMA_NAME (1226) +#define SQL_CA_SS_TYPE_NAME (1227) // Include logger bridge for LOG macros #include "logger_bridge.hpp" @@ -84,6 +92,8 @@ typedef SQLRETURN(SQL_API* SQLExecuteFunc)(SQLHANDLE); typedef SQLRETURN(SQL_API* SQLRowCountFunc)(SQLHSTMT, SQLLEN*); typedef SQLRETURN(SQL_API* SQLSetDescFieldFunc)(SQLHDESC, SQLSMALLINT, SQLSMALLINT, SQLPOINTER, SQLINTEGER); +typedef SQLRETURN(SQL_API* SQLGetDescFieldFunc)(SQLHDESC, SQLSMALLINT, SQLSMALLINT, SQLPOINTER, + SQLINTEGER, SQLINTEGER*); typedef SQLRETURN(SQL_API* SQLGetStmtAttrFunc)(SQLHSTMT, SQLINTEGER, SQLPOINTER, SQLINTEGER, SQLINTEGER*); @@ -169,6 +179,7 @@ extern SQLBindParameterFunc SQLBindParameter_ptr; extern SQLExecuteFunc SQLExecute_ptr; extern SQLRowCountFunc SQLRowCount_ptr; extern SQLSetDescFieldFunc SQLSetDescField_ptr; +extern SQLGetDescFieldFunc SQLGetDescField_ptr; extern SQLGetStmtAttrFunc SQLGetStmtAttr_ptr; // Data retrieval APIs @@ -280,6 +291,19 @@ struct DescribedParamInfo { SQLSMALLINT decimalDigits; }; +struct TvpColumnInfo { + SQLSMALLINT sqlType; + SQLULEN columnSize; + SQLSMALLINT decimalDigits; +}; + +struct TvpParamInfo { + std::u16string catalog; + std::u16string schema; + std::u16string typeName; + std::vector columns; +}; + class SqlHandle { public: SqlHandle(SQLSMALLINT type, SQLHANDLE rawHandle); @@ -317,7 +341,11 @@ class SqlHandle { // on handle free. No mutex needed — ODBC statement handles are not // thread-safe by spec (same assumption as the rest of the driver). std::unordered_map describeCache; - void clearDescribeCache() { describeCache.clear(); } + std::unordered_map tvpCache; + void clearDescribeCache() { + describeCache.clear(); + tvpCache.clear(); + } private: SQLSMALLINT _type; diff --git a/mssql_python/pybind/param_detect.hpp b/mssql_python/pybind/param_detect.hpp index 172a98b51..feb0aaadc 100644 --- a/mssql_python/pybind/param_detect.hpp +++ b/mssql_python/pybind/param_detect.hpp @@ -51,9 +51,11 @@ struct ParamInfo { SQLSMALLINT paramCType = SQL_C_DEFAULT; SQLSMALLINT paramSQLType = SQL_UNKNOWN_TYPE; SQLULEN columnSize = 0; + SQLULEN bufferSize = 0; SQLSMALLINT decimalDigits = 0; SQLLEN strLenOrInd = 0; // Required for DAE bool isDAE = false; // Indicates if we need to stream + bool isTVP = false; // Strong reference to the Python object for DAE (data-at-execution) streaming. // py::object owns the refcount, so the compiler-generated destructor, copy and // move operations are all correct and this struct needs no rule-of-five. @@ -310,14 +312,27 @@ inline std::vector DetectParamTypes(PyObject* params, PyObject* input ParamInfo& info = infos[i]; info.inputOutputType = SQL_PARAM_INPUT; info.isDAE = false; + info.isTVP = false; + + PyObject* obj = PyList_GET_ITEM(params, i); + if (PyList_Check(obj) || PyTuple_Check(obj)) { + if (i < inputSizeCount) { + throw py::type_error("setinputsizes cannot override table-valued parameters"); + } + info.paramSQLType = SQL_SS_TABLE; + info.paramCType = SQL_C_BINARY; + info.columnSize = static_cast(PySequence_Size(obj)); + info.decimalDigits = 0; + info.isTVP = true; + info.dataPtr = borrow(obj); + continue; + } if (i < inputSizeCount) { ApplyInputSizeOverride(params, PyList_GET_ITEM(inputSizes, i), i, info); continue; } - PyObject* obj = PyList_GET_ITEM(params, i); - // --- None --- if (obj == Py_None) { info.paramSQLType = SQL_UNKNOWN_TYPE; diff --git a/tests/test_004_cursor.py b/tests/test_004_cursor.py index 219d4833a..ab2fde2ad 100644 --- a/tests/test_004_cursor.py +++ b/tests/test_004_cursor.py @@ -1879,6 +1879,7 @@ def test_executemany_unicode_and_empty_strings(cursor, db_connection): (5, ""), (6, "Ñice tëxt"), (7, ""), + (8, "a\x00b"), ] # Execute the batch insert @@ -1890,7 +1891,7 @@ def test_executemany_unicode_and_empty_strings(cursor, db_connection): results = cursor.fetchall() # Check that we got the right number of rows - assert len(results) == 7, f"Expected 7 rows, got {len(results)}" + assert len(results) == 8, f"Expected 8 rows, got {len(results)}" # Check each row for i, (actual, expected_row) in enumerate(zip(results, test_data)): diff --git a/tests/test_028_table_valued_parameters.py b/tests/test_028_table_valued_parameters.py new file mode 100644 index 000000000..4ce1c40a6 --- /dev/null +++ b/tests/test_028_table_valued_parameters.py @@ -0,0 +1,383 @@ +""" +Integration coverage for SQL Server table-valued parameters. +""" + +import datetime +import uuid +from decimal import Decimal + +import pytest + +from mssql_python import ( + DatabaseError, + NotSupportedError, + SQL_INTEGER, + SQL_WVARCHAR, + ddbc_bindings, +) +from mssql_python.odbc_provider import ProviderManager + + +@pytest.fixture +def tvp_procedure(cursor, db_connection): + suffix = uuid.uuid4().hex + type_name = f"pytest_tvp_type_{suffix}" + wildcard_match_type_name = type_name.replace("_", "X") + procedure_name = f"pytest_tvp_proc_{suffix}" + + try: + cursor.execute( + f"CREATE TYPE dbo.[{type_name}] AS TABLE " "(id int NOT NULL, label nvarchar(100) NULL)" + ) + cursor.execute( + f"CREATE TYPE dbo.[{wildcard_match_type_name}] AS TABLE " "(decoy_id bigint NOT NULL)" + ) + cursor.execute(f""" + CREATE PROCEDURE dbo.[{procedure_name}] + @prefix int, + @items dbo.[{type_name}] READONLY, + @suffix nvarchar(20) + AS + BEGIN + SET NOCOUNT ON; + SELECT id + @prefix, label + @suffix + FROM @items + ORDER BY id; + END + """) + db_connection.commit() + yield type_name, procedure_name + finally: + cursor.execute(f"DROP PROCEDURE IF EXISTS dbo.[{procedure_name}]") + cursor.execute(f"DROP TYPE IF EXISTS dbo.[{type_name}]") + cursor.execute(f"DROP TYPE IF EXISTS dbo.[{wildcard_match_type_name}]") + db_connection.commit() + + +@pytest.fixture +def typed_tvp_procedure(cursor, db_connection): + suffix = uuid.uuid4().hex + type_name = f"pytest_typed_tvp_type_{suffix}" + procedure_name = f"pytest_typed_tvp_proc_{suffix}" + + try: + cursor.execute(f""" + CREATE TYPE dbo.[{type_name}] AS TABLE ( + bit_value bit NULL, + tiny_value tinyint NULL, + small_value smallint NULL, + int_value int NULL, + big_value bigint NULL, + float_value float NULL, + decimal_value decimal(20, 6) NULL, + text_value nvarchar(100) NULL, + binary_value varbinary(100) NULL, + date_value date NULL, + time_value time(6) NULL, + datetime_value datetime2(6) NULL, + offset_value datetimeoffset(6) NULL, + guid_value uniqueidentifier NULL, + all_null_binary varbinary(16) NULL + ) + """) + cursor.execute(f""" + CREATE PROCEDURE dbo.[{procedure_name}] + @items dbo.[{type_name}] READONLY + AS + BEGIN + SET NOCOUNT ON; + SELECT * + FROM @items + ORDER BY int_value; + END + """) + db_connection.commit() + yield procedure_name + finally: + cursor.execute(f"DROP PROCEDURE IF EXISTS dbo.[{procedure_name}]") + cursor.execute(f"DROP TYPE IF EXISTS dbo.[{type_name}]") + db_connection.commit() + + +@pytest.fixture +def multiple_tvp_procedure(cursor, db_connection): + suffix = uuid.uuid4().hex + type_name = f"pytest_multi_tvp_type_{suffix}" + procedure_name = f"pytest_multi_tvp_proc_{suffix}" + + try: + cursor.execute(f"CREATE TYPE dbo.[{type_name}] AS TABLE (id int NOT NULL)") + cursor.execute(f""" + CREATE PROCEDURE dbo.[{procedure_name}] + @left_rows dbo.[{type_name}] READONLY, + @offset int, + @right_rows dbo.[{type_name}] READONLY + AS + BEGIN + SET NOCOUNT ON; + SELECT side, id + @offset + FROM ( + SELECT 'left' AS side, id FROM @left_rows + UNION ALL + SELECT 'right', id FROM @right_rows + ) AS combined + ORDER BY side, id; + END + """) + db_connection.commit() + yield procedure_name + finally: + cursor.execute(f"DROP PROCEDURE IF EXISTS dbo.[{procedure_name}]") + cursor.execute(f"DROP TYPE IF EXISTS dbo.[{type_name}]") + db_connection.commit() + + +@pytest.fixture +def schema_qualified_large_tvp(cursor, db_connection): + suffix = uuid.uuid4().hex + schema_name = f"pytest_tvp_schema_{suffix}" + type_name = f"pytest_large_tvp_type_{suffix}" + procedure_name = f"pytest_large_tvp_proc_{suffix}" + + try: + cursor.execute(f"CREATE SCHEMA [{schema_name}]") + cursor.execute(f""" + CREATE TYPE [{schema_name}].[{type_name}] AS TABLE ( + text_value nvarchar(max) NULL, + binary_value varbinary(max) NULL + ) + """) + cursor.execute(f""" + CREATE PROCEDURE [{schema_name}].[{procedure_name}] + @items [{schema_name}].[{type_name}] READONLY + AS + BEGIN + SET NOCOUNT ON; + SELECT text_value, binary_value FROM @items; + END + """) + db_connection.commit() + yield schema_name, procedure_name + finally: + cursor.execute(f"DROP PROCEDURE IF EXISTS [{schema_name}].[{procedure_name}]") + cursor.execute(f"DROP TYPE IF EXISTS [{schema_name}].[{type_name}]") + cursor.execute(f"DROP SCHEMA IF EXISTS [{schema_name}]") + db_connection.commit() + + +def test_tvp_positional_named_empty_and_statement_reuse(cursor, tvp_procedure): + _, procedure_name = tvp_procedure + sql = f"EXEC dbo.[{procedure_name}] ?, ?, ?" + + cursor.execute(sql, (10, [(1, "Keyboard"), (2, None)], "!")) + assert [tuple(row) for row in cursor.fetchall()] == [(11, "Keyboard!"), (12, None)] + + cursor.execute(sql, (10, [], "!")) + assert cursor.fetchall() == [] + + cursor.execute(sql, (None, [(3, "Nullable")], "")) + assert tuple(cursor.fetchone()) == (None, "Nullable") + + cursor.execute( + f"EXEC dbo.[{procedure_name}] %(prefix)s, %(items)s, %(suffix)s", + {"prefix": 0, "items": [(4, "Mouse")], "suffix": "?"}, + ) + assert tuple(cursor.fetchone()) == (4, "Mouse?") + + +def test_tvp_empty_first_and_all_null_column(cursor, tvp_procedure): + _, procedure_name = tvp_procedure + sql = f"EXEC dbo.[{procedure_name}] ?, ?, ?" + + cursor.execute(sql, (0, [], "")) + assert cursor.fetchall() == [] + + cursor.execute(sql, (0, [(1, None), (2, None)], "")) + assert [tuple(row) for row in cursor.fetchall()] == [(1, None), (2, None)] + + +def test_tvp_binds_supported_cell_types(cursor, typed_tvp_procedure): + first_guid = uuid.uuid4() + second_guid = uuid.uuid4() + offset = datetime.timezone(datetime.timedelta(hours=5, minutes=30)) + rows = [ + ( + True, + 255, + -32768, + -2147483648, + -(2**63), + 1.25, + Decimal("-12345678901234.567890"), + "Grüße 😀", + b"\x00\xff", + datetime.date(2026, 9, 15), + datetime.time(12, 34, 56, 123456), + datetime.datetime(2026, 9, 15, 12, 34, 56, 123456), + datetime.datetime(2026, 9, 15, 12, 34, 56, 123456, tzinfo=offset), + first_guid, + None, + ), + ( + False, + 0, + 32767, + 2147483647, + 2**63 - 1, + -2.5, + Decimal("12.345"), + "later row is longer", + b"\x01\x02\x03", + datetime.date(2026, 9, 16), + datetime.time(1, 2, 3, 4), + datetime.datetime(2026, 9, 16, 1, 2, 3, 4), + datetime.datetime(2026, 9, 16, 1, 2, 3, 4, tzinfo=datetime.timezone.utc), + second_guid, + None, + ), + ] + + cursor.execute(f"EXEC dbo.[{typed_tvp_procedure}] ?", (rows,)) + actual = [tuple(row) for row in cursor.fetchall()] + + assert actual[0][:9] == rows[0][:9] + assert actual[1][:9] == rows[1][:9] + assert actual[0][9:13] == rows[0][9:13] + assert actual[1][9:13] == rows[1][9:13] + assert str(actual[0][13]) == str(first_guid) + assert str(actual[1][13]) == str(second_guid) + assert actual[0][14] is None + assert actual[1][14] is None + + invalid_row = list(rows[0]) + invalid_row[6] = Decimal("NaN") + with pytest.raises(ValueError, match="non-finite Decimal"): + cursor.execute( + f"EXEC dbo.[{typed_tvp_procedure}] ?", + ([tuple(invalid_row)],), + ) + + +def test_multiple_tvps_with_interleaved_scalar(cursor, multiple_tvp_procedure): + cursor.execute( + f"EXEC dbo.[{multiple_tvp_procedure}] ?, ?, ?", + ([(1,), (2,)], 10, [(3,), (4,)]), + ) + assert [tuple(row) for row in cursor.fetchall()] == [ + ("left", 11), + ("left", 12), + ("right", 13), + ("right", 14), + ] + + +def test_schema_qualified_tvp_with_large_and_embedded_null_values( + cursor, schema_qualified_large_tvp +): + schema_name, procedure_name = schema_qualified_large_tvp + large_text = "prefix\x00suffix" + "😀" * 2500 + large_binary = b"\x00\xff" * 4501 + rows = ((large_text, large_binary), (None, None)) + + cursor.execute(f"EXEC [{schema_name}].[{procedure_name}] ?", (rows,)) + actual = [tuple(row) for row in cursor.fetchall()] + + assert actual == [(large_text, large_binary), (None, None)] + + +def test_failed_tvp_does_not_poison_cursor(cursor, tvp_procedure): + _, procedure_name = tvp_procedure + sql = f"EXEC dbo.[{procedure_name}] ?, ?, ?" + + with pytest.raises(DatabaseError): + cursor.execute(sql, (0, [(None, "invalid")], "")) + + cursor.execute("SELECT ?", 42) + assert cursor.fetchone()[0] == 42 + + cursor.execute(sql, (0, [(1, "valid")], "")) + assert tuple(cursor.fetchone()) == (1, "valid") + + +def test_tvp_rejects_invalid_shape_and_cells(cursor, tvp_procedure): + _, procedure_name = tvp_procedure + sql = f"EXEC dbo.[{procedure_name}] ?, ?, ?" + + with pytest.raises(ValueError, match="same number"): + cursor.execute(sql, (0, [(1, "valid"), (2,)], "")) + with pytest.raises(TypeError, match="incompatible Python types"): + cursor.execute(sql, (0, [(1, "valid"), ("2", "invalid")], "")) + with pytest.raises(TypeError, match="only be bound"): + cursor.execute("SELECT CAST(? AS int)", ([(1,)],)) + + +def test_tvp_rejected_by_executemany_before_first_write(cursor, tvp_procedure): + cursor.execute("CREATE TABLE #pytest_tvp_executemany (id int)") + + with pytest.raises(NotSupportedError): + cursor.executemany( + "INSERT INTO #pytest_tvp_executemany VALUES (?)", + [(1,), ([(2, "label")],)], + ) + + cursor.execute("SELECT COUNT(*) FROM #pytest_tvp_executemany") + assert cursor.fetchone()[0] == 0 + + +def test_tvp_rejected_with_setinputsizes(cursor, tvp_procedure): + _, procedure_name = tvp_procedure + cursor.setinputsizes( + [ + (SQL_INTEGER, 10, 0), + (SQL_WVARCHAR, 100, 0), + (SQL_WVARCHAR, 20, 0), + ] + ) + try: + with pytest.raises(NotSupportedError): + cursor.execute( + f"EXEC dbo.[{procedure_name}] ?, ?, ?", + (0, [(1, "label")], ""), + ) + finally: + cursor.setinputsizes(None) + + +def test_tvp_rejected_by_mssql_odbc_provider(cursor, tvp_procedure, monkeypatch): + _, procedure_name = tvp_procedure + monkeypatch.setattr(ProviderManager, "effective", lambda *_: "mssql-odbc") + + with pytest.raises(NotSupportedError, match="mssql-odbc provider"): + cursor.execute( + f"EXEC dbo.[{procedure_name}] ?, ?, ?", + (0, [(1, "label")], ""), + ) + + +def test_tvp_metadata_handle_inherits_query_timeout(db_connection, tvp_procedure, monkeypatch): + _, procedure_name = tvp_procedure + real_set_stmt_attr = ddbc_bindings.DDBCSQLSetStmtAttr + timeout_handles = [] + + def record_timeout(statement_handle, attribute, value): + timeout_handles.append(statement_handle) + return real_set_stmt_attr(statement_handle, attribute, value) + + db_connection.timeout = 2 + monkeypatch.setattr( + ddbc_bindings, + "DDBCSQLSetStmtAttr", + record_timeout, + ) + try: + with db_connection.cursor() as timeout_cursor: + timeout_cursor.execute( + f"EXEC dbo.[{procedure_name}] ?, ?, ?", + (0, [(1, "label")], ""), + ) + assert tuple(timeout_cursor.fetchone()) == (1, "label") + finally: + db_connection.timeout = 0 + + assert len(timeout_handles) == 2 + assert timeout_handles[0] is not timeout_handles[1] From 635d19897edbbf0b1eaa64eae6a04fa48b77c874 Mon Sep 17 00:00:00 2001 From: Gaurav Sharma <223556219+Copilot@users.noreply.github.com> Date: Tue, 15 Sep 2026 17:09:44 +0530 Subject: [PATCH 02/15] CHORE: Link TVP support to ADO work item Replace the stale GitHub issue reference with the ADO task that tracks this feature. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e4eecb9a0..2e940ff99 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ## [Unreleased] ### Added -- **GH-319:** Added input table-valued parameter support for stored procedure +- **AB#48177:** Added input table-valued parameter support for stored procedure calls. Pass a materialized list or tuple of rows as one `execute()` parameter, including empty tables and positional or named parameters. The Rust ODBC provider does not yet support TVPs. From 71c14b967533036b0c72de8e076f0f6b001bc4e7 Mon Sep 17 00:00:00 2001 From: Gaurav Sharma <223556219+Copilot@users.noreply.github.com> Date: Tue, 15 Sep 2026 18:09:59 +0530 Subject: [PATCH 03/15] REFACTOR: Deduplicate TVP parameter binding Reuse scalar type detection and parameter metadata for TVP columns, and remove redundant sizing and metadata ordering work. AB#48177 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- mssql_python/pybind/ddbc_bindings.cpp | 126 ++++++-------------------- mssql_python/pybind/ddbc_bindings.h | 8 +- 2 files changed, 29 insertions(+), 105 deletions(-) diff --git a/mssql_python/pybind/ddbc_bindings.cpp b/mssql_python/pybind/ddbc_bindings.cpp index bcc293752..6e49aae05 100644 --- a/mssql_python/pybind/ddbc_bindings.cpp +++ b/mssql_python/pybind/ddbc_bindings.cpp @@ -26,7 +26,7 @@ SQLRETURN BindParameterArray(SqlHandle& handle, SQLHANDLE hStmt, const py::list& columnwise_params, std::vector& paramInfos, size_t paramSetSize, std::vector>& paramBuffers, - const std::string& charEncoding, bool resolveUnknownTypes = true); + const std::string& charEncoding); //------------------------------------------------------------------------------------------------- // Macro definitions @@ -469,9 +469,6 @@ static ParamInfo MergeTableColumnInfo(const py::list& values, bool hasValue = false; bool integerColumn = false; bool textColumn = false; - bool hasInteger = false; - int64_t minInteger = 0; - int64_t maxInteger = 0; SQLULEN maxSize = 0; SQLULEN maxIntegerDigits = 0; SQLSMALLINT maxScale = 0; @@ -494,40 +491,20 @@ static ParamInfo MergeTableColumnInfo(const py::list& values, textColumn = currentText; hasValue = true; } else if (integerColumn && currentInteger) { - // Integer widths are merged below from the full observed range. - } else if (textColumn && currentText) { - if (current.paramSQLType == SQL_WVARCHAR) { - merged.paramSQLType = SQL_WVARCHAR; + // Scalar detection already range-checks integers; retain its widest type. + if (current.columnSize > merged.columnSize) { + merged = current; } - } else if (merged.paramCType != current.paramCType || - merged.paramSQLType != current.paramSQLType) { + } else if (!(textColumn && currentText) && + (merged.paramCType != current.paramCType || + merged.paramSQLType != current.paramSQLType)) { throw py::type_error("TVP column " + std::to_string(columnIndex) + " contains incompatible Python types at row " + std::to_string(rowIndex)); } if (currentInteger) { - int overflow = 0; - int64_t value = PyLong_AsLongLongAndOverflow(values[rowIndex].ptr(), &overflow); - if (overflow != 0) { - PyErr_Clear(); - throw py::value_error("TVP integer cell is out of range for SQL BIGINT"); - } - if (PyErr_Occurred()) - throw py::error_already_set(); - if (!integerColumn) { - throw py::type_error("TVP column " + std::to_string(columnIndex) + - " contains incompatible Python types at row " + - std::to_string(rowIndex)); - } - if (!hasInteger) { - minInteger = value; - maxInteger = value; - hasInteger = true; - } else { - minInteger = std::min(minInteger, value); - maxInteger = std::max(maxInteger, value); - } + continue; } else if (currentText) { SQLULEN size = current.isDAE ? static_cast(current.utf16Len) : current.columnSize; @@ -559,37 +536,11 @@ static ParamInfo MergeTableColumnInfo(const py::list& values, } if (integerColumn) { - if (minInteger >= 0 && maxInteger <= UINT8_MAX) { + if (merged.paramCType == SQL_C_TINYINT) { merged.paramCType = SQL_C_UTINYINT; - merged.paramSQLType = SQL_TINYINT; - merged.columnSize = 3; - } else if (minInteger >= INT16_MIN && maxInteger <= INT16_MAX) { - merged.paramCType = SQL_C_SHORT; - merged.paramSQLType = SQL_SMALLINT; - merged.columnSize = 5; - } else if (minInteger >= INT32_MIN && maxInteger <= INT32_MAX) { - merged.paramCType = SQL_C_LONG; - merged.paramSQLType = SQL_INTEGER; - merged.columnSize = 10; - } else { - merged.paramCType = SQL_C_SBIGINT; - merged.paramSQLType = SQL_BIGINT; - merged.columnSize = 19; } - merged.decimalDigits = 0; - } else if (textColumn) { + } else if (textColumn || merged.paramCType == SQL_C_BINARY) { merged.columnSize = std::max(maxSize, 1); - if (merged.paramSQLType == SQL_WVARCHAR && maxSize > MAX_INLINE_CHAR) { - merged.paramSQLType = SQL_WLONGVARCHAR; - } else if (merged.paramSQLType == SQL_VARCHAR && maxSize > MAX_INLINE_BINARY) { - merged.paramSQLType = SQL_LONGVARCHAR; - } - merged.isDAE = false; - } else if (merged.paramCType == SQL_C_BINARY) { - merged.columnSize = std::max(maxSize, 1); - if (maxSize > MAX_INLINE_BINARY) { - merged.paramSQLType = SQL_LONGVARBINARY; - } merged.isDAE = false; } else if (merged.paramCType == SQL_C_NUMERIC) { merged.decimalDigits = maxScale; @@ -656,7 +607,7 @@ static bool NormalizeTableDecimalColumn(py::list& values, ParamInfo& info, size_ } static std::vector -DetectTableColumnTypes(py::list& columnValues, const std::vector& declaredColumns) { +DetectTableColumnTypes(py::list& columnValues, const std::vector& declaredColumns) { std::vector columnInfos(columnValues.size()); for (size_t columnIndex = 0; columnIndex < columnValues.size(); ++columnIndex) { py::list values = columnValues[columnIndex].cast(); @@ -717,7 +668,7 @@ class TableMetadataScope { SQLHANDLE handle; }; -static std::vector LoadTableColumnMetadata(const SqlHandlePtr& metadataHandle, +static std::vector LoadTableColumnMetadata(const SqlHandlePtr& metadataHandle, const std::u16string& catalog, const std::u16string& schema, const std::u16string& typeName) { @@ -757,7 +708,7 @@ static std::vector LoadTableColumnMetadata(const SqlHandlePtr& me ThrowTableMetadataError(metadataHandle, rc, "Failed to discover TVP columns"); } - std::vector> orderedColumns; + std::vector columns; while (true) { { py::gil_scoped_release release; @@ -773,7 +724,6 @@ static std::vector LoadTableColumnMetadata(const SqlHandlePtr& me SQLSMALLINT sqlType = SQL_UNKNOWN_TYPE; SQLINTEGER columnSize = 0; SQLSMALLINT decimalDigits = 0; - SQLINTEGER ordinal = 0; SQLLEN indicator = 0; rc = SQLGetData_ptr(hMetadataStmt, 5, SQL_C_SSHORT, &sqlType, sizeof(sqlType), &indicator); @@ -789,31 +739,18 @@ static std::vector LoadTableColumnMetadata(const SqlHandlePtr& me rc = SQL_SUCCESS; } } - if (SQL_SUCCEEDED(rc)) { - rc = SQLGetData_ptr(hMetadataStmt, 17, SQL_C_SLONG, &ordinal, sizeof(ordinal), - &indicator); - } if (!SQL_SUCCEEDED(rc)) { ThrowTableMetadataError(metadataHandle, rc, "Failed to read TVP column metadata"); } - orderedColumns.push_back({ - ordinal, - {sqlType, static_cast(std::max(columnSize, 0)), decimalDigits}, + columns.push_back({ + sqlType, static_cast(std::max(columnSize, 0)), decimalDigits, }); } - if (orderedColumns.empty()) { + if (columns.empty()) { throw std::runtime_error("No column metadata was returned for the table type"); } - std::sort(orderedColumns.begin(), orderedColumns.end(), - [](const auto& left, const auto& right) { return left.first < right.first; }); - - std::vector columns; - columns.reserve(orderedColumns.size()); - for (const auto& entry : orderedColumns) { - columns.push_back(entry.second); - } return columns; } @@ -878,9 +815,6 @@ static SQLRETURN ResolveTableValuedParamTypes(SqlHandle& handle, SQLHANDLE hStmt LoadTableColumnMetadata(metadataHandle, catalog, schema, typeName), }; } - info.paramSQLType = SQL_SS_TABLE; - info.paramCType = SQL_C_BINARY; - info.decimalDigits = 0; } return SQL_SUCCESS; } @@ -900,7 +834,7 @@ static SQLRETURN BindTableValuedParameter(SqlHandle& handle, SQLHANDLE hStmt, in throw std::runtime_error("TVP column metadata was not resolved before binding"); } const TvpParamInfo& tableMetadata = metadataEntry->second; - const std::vector& declaredColumns = tableMetadata.columns; + const std::vector& declaredColumns = tableMetadata.columns; if (!rows.empty()) { const py::handle firstRow = rows[0]; if (!PyList_Check(firstRow.ptr()) && !PyTuple_Check(firstRow.ptr())) { @@ -934,7 +868,7 @@ static SQLRETURN BindTableValuedParameter(SqlHandle& handle, SQLHANDLE hStmt, in columnInfos = DetectTableColumnTypes(columnValues, declaredColumns); for (size_t columnIndex = 0; columnIndex < columnCount; ++columnIndex) { ParamInfo& inferred = columnInfos[columnIndex]; - const TvpColumnInfo& declared = declaredColumns[columnIndex]; + const DescribedParamInfo& declared = declaredColumns[columnIndex]; inferred.paramSQLType = declared.sqlType; if (declared.columnSize == 0) { if (declared.sqlType == SQL_WVARCHAR && inferred.columnSize > MAX_INLINE_CHAR) { @@ -996,7 +930,7 @@ static SQLRETURN BindTableValuedParameter(SqlHandle& handle, SQLHANDLE hStmt, in SQLRETURN bindRc = SQL_ERROR; try { bindRc = BindParameterArray(handle, hStmt, columnValues, columnInfos, rows.size(), - paramBuffers, charEncoding, false); + paramBuffers, charEncoding); } catch (...) { SQLSetStmtAttr_ptr(hStmt, SQL_SOPT_SS_PARAM_FOCUS, nullptr, SQL_IS_INTEGER); throw; @@ -2775,7 +2709,7 @@ SQLRETURN SQLExecute_wrap(const SqlHandlePtr statementHandle, SQLRETURN BindParameterArray(SqlHandle& handle, SQLHANDLE hStmt, const py::list& columnwise_params, std::vector& paramInfos, size_t paramSetSize, std::vector>& paramBuffers, - const std::string& charEncoding, bool resolveUnknownTypes) { + const std::string& charEncoding) { PERF_TIMER("BindParameterArray"); LOG("BindParameterArray: Starting column-wise array binding - " "param_count=%zu, param_set_size=%zu", @@ -2785,9 +2719,7 @@ SQLRETURN BindParameterArray(SqlHandle& handle, SQLHANDLE hStmt, const py::list& try { // GH-627: resolve unknown NULL array param SQL types before binding any param. - if (resolveUnknownTypes) { - PreResolveUnknownNullTypes(handle, hStmt, paramInfos); - } + PreResolveUnknownNullTypes(handle, hStmt, paramInfos); for (int paramIndex = 0; paramIndex < columnwise_params.size(); ++paramIndex) { const py::list& columnValues = columnwise_params[paramIndex].cast(); ParamInfo& info = paramInfos[paramIndex]; @@ -2846,29 +2778,27 @@ SQLRETURN BindParameterArray(SqlHandle& handle, SQLHANDLE hStmt, const py::list& break; } case SQL_C_WCHAR: { - const SQLULEN valueBufferSize = - info.bufferSize > 0 ? info.bufferSize : info.columnSize; LOG("BindParameterArray: Binding SQL_C_WCHAR array - " "param_index=%d, count=%zu, column_size=%zu", - paramIndex, paramSetSize, valueBufferSize); + paramIndex, paramSetSize, info.columnSize); SQLWCHAR* wcharArray = AllocateParamBufferArray( - tempBuffers, paramSetSize * (valueBufferSize + 1)); + tempBuffers, paramSetSize * (info.columnSize + 1)); strLenOrIndArray = AllocateParamBufferArray(tempBuffers, paramSetSize); for (size_t i = 0; i < paramSetSize; ++i) { if (columnValues[i].is_none()) { strLenOrIndArray[i] = SQL_NULL_DATA; - std::memset(wcharArray + i * (valueBufferSize + 1), 0, - (valueBufferSize + 1) * sizeof(SQLWCHAR)); + std::memset(wcharArray + i * (info.columnSize + 1), 0, + (info.columnSize + 1) * sizeof(SQLWCHAR)); } else { std::u16string wstr = columnValues[i].cast(); // u16string is already UTF-16, so the // original check is sufficient - if (wstr.length() > valueBufferSize) { + if (wstr.length() > info.columnSize) { ThrowStdException("Input string exceeds allowed column size " "at parameter index " + std::to_string(paramIndex)); } - std::memcpy(wcharArray + i * (valueBufferSize + 1), wstr.c_str(), + std::memcpy(wcharArray + i * (info.columnSize + 1), wstr.c_str(), (wstr.length() + 1) * sizeof(SQLWCHAR)); strLenOrIndArray[i] = static_cast(wstr.length() * sizeof(SQLWCHAR)); @@ -2878,7 +2808,7 @@ SQLRETURN BindParameterArray(SqlHandle& handle, SQLHANDLE hStmt, const py::list& "param_index=%d", paramIndex); dataPtr = wcharArray; - bufferLength = (valueBufferSize + 1) * sizeof(SQLWCHAR); + bufferLength = (info.columnSize + 1) * sizeof(SQLWCHAR); break; } case SQL_C_TINYINT: diff --git a/mssql_python/pybind/ddbc_bindings.h b/mssql_python/pybind/ddbc_bindings.h index 3558db178..991de8559 100644 --- a/mssql_python/pybind/ddbc_bindings.h +++ b/mssql_python/pybind/ddbc_bindings.h @@ -291,17 +291,11 @@ struct DescribedParamInfo { SQLSMALLINT decimalDigits; }; -struct TvpColumnInfo { - SQLSMALLINT sqlType; - SQLULEN columnSize; - SQLSMALLINT decimalDigits; -}; - struct TvpParamInfo { std::u16string catalog; std::u16string schema; std::u16string typeName; - std::vector columns; + std::vector columns; }; class SqlHandle { From d4e245160010e3c69f5896a9a4a7ca123b787f44 Mon Sep 17 00:00:00 2001 From: Gaurav Sharma Date: Tue, 15 Sep 2026 19:28:03 +0530 Subject: [PATCH 04/15] FIX: Preserve TVP conversion precision and diagnostics Preserve source metadata and exact Decimal values during TVP binding so SQL Server can apply destination conversions. Retain binding diagnostics during parameter-focus cleanup and route timeout diagnostics to the selected statement handle. Add regression coverage for conversions, buffer sizing, validation, and recovery. AB#48177 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- mssql_python/cursor.py | 5 +- mssql_python/pybind/ddbc_bindings.cpp | 165 +++++++---------- tests/test_004_cursor_timeout.py | 75 ++++++++ tests/test_028_table_valued_parameters.py | 212 ++++++++++++++++++++++ 4 files changed, 360 insertions(+), 97 deletions(-) create mode 100644 tests/test_004_cursor_timeout.py diff --git a/mssql_python/cursor.py b/mssql_python/cursor.py index 8d2d7d56e..34f21ecb7 100644 --- a/mssql_python/cursor.py +++ b/mssql_python/cursor.py @@ -1008,12 +1008,13 @@ def _set_timeout(self, statement_handle=None) -> None: logger.debug("_set_timeout: Setting query timeout=%d seconds", self._timeout) try: timeout_value = int(self._timeout) + target_handle = statement_handle or self.hstmt ret = ddbc_bindings.DDBCSQLSetStmtAttr( - statement_handle or self.hstmt, + target_handle, ddbc_sql_const.SQL_ATTR_QUERY_TIMEOUT.value, timeout_value, ) - check_error(ddbc_sql_const.SQL_HANDLE_STMT.value, self.hstmt, ret) + check_error(ddbc_sql_const.SQL_HANDLE_STMT.value, target_handle, ret) logger.debug("Query timeout set to %d seconds", timeout_value) except Exception as e: # pylint: disable=broad-exception-caught logger.warning("Failed to set query timeout: %s", str(e)) diff --git a/mssql_python/pybind/ddbc_bindings.cpp b/mssql_python/pybind/ddbc_bindings.cpp index 6e49aae05..94af75ef0 100644 --- a/mssql_python/pybind/ddbc_bindings.cpp +++ b/mssql_python/pybind/ddbc_bindings.cpp @@ -506,6 +506,9 @@ static ParamInfo MergeTableColumnInfo(const py::list& values, if (currentInteger) { continue; } else if (currentText) { + if (current.paramSQLType == SQL_WVARCHAR) { + merged.paramSQLType = SQL_WVARCHAR; + } SQLULEN size = current.isDAE ? static_cast(current.utf16Len) : current.columnSize; maxSize = std::max(maxSize, size); @@ -553,71 +556,40 @@ static ParamInfo MergeTableColumnInfo(const py::list& values, return merged; } -static bool NormalizeTableDecimalColumn(py::list& values, ParamInfo& info, size_t columnIndex) { - bool hasDecimal = false; - bool hasOtherValue = false; - PyObject* decimalType = PyTypeCache::get_decimal_class(); - for (const py::handle value : values) { - if (value.is_none()) { - continue; - } - int isDecimal = PyObject_IsInstance(value.ptr(), decimalType); - if (isDecimal == -1) - throw py::error_already_set(); - hasDecimal = hasDecimal || isDecimal == 1; - hasOtherValue = hasOtherValue || isDecimal == 0; - } - if (!hasDecimal) { - return false; - } - if (hasOtherValue) { - throw py::type_error("TVP column " + std::to_string(columnIndex) + - " contains incompatible Python types"); - } - - py::str formatSpec("f"); - SQLULEN maxSize = 1; - for (size_t rowIndex = 0; rowIndex < values.size(); ++rowIndex) { - if (values[rowIndex].is_none()) { - continue; - } - py::object isFinite = - steal(PyObject_CallMethod(values[rowIndex].ptr(), "is_finite", nullptr)); - if (!isFinite) - throw py::error_already_set(); - int finite = PyObject_IsTrue(isFinite.ptr()); - if (finite == -1) - throw py::error_already_set(); - if (finite == 0) { - throw py::value_error("Cannot bind non-finite Decimal (NaN/Infinity) in a TVP"); - } - py::object formatted = steal(PyObject_Format(values[rowIndex].ptr(), formatSpec.ptr())); - if (!formatted) - throw py::error_already_set(); - maxSize = std::max(maxSize, static_cast(PyUnicode_GET_LENGTH(formatted.ptr()))); - values[rowIndex] = std::move(formatted); - } - - info.paramCType = SQL_C_CHAR; - info.paramSQLType = SQL_NUMERIC; - info.columnSize = maxSize; - info.bufferSize = maxSize; - info.decimalDigits = 0; - return true; -} - -static std::vector -DetectTableColumnTypes(py::list& columnValues, const std::vector& declaredColumns) { +static std::vector DetectTableColumnTypes(py::list& columnValues) { std::vector columnInfos(columnValues.size()); for (size_t columnIndex = 0; columnIndex < columnValues.size(); ++columnIndex) { py::list values = columnValues[columnIndex].cast(); - if ((declaredColumns[columnIndex].sqlType == SQL_DECIMAL || - declaredColumns[columnIndex].sqlType == SQL_NUMERIC) && - NormalizeTableDecimalColumn(values, columnInfos[columnIndex], columnIndex)) { - continue; + py::list detectedValues = values.attr("copy")(); + std::vector valueInfos = DetectParamTypes(detectedValues.ptr(), Py_None); + ParamInfo& info = columnInfos[columnIndex]; + info = MergeTableColumnInfo(detectedValues, valueInfos, columnIndex); + if (info.paramCType == SQL_C_NUMERIC) { + if (info.columnSize > MAX_NUMERIC_PRECISION) { + throw py::value_error("TVP Decimal column requires precision greater than 38"); + } + // Preserve each coefficient's scale using exact text, while binding a common + // source NUMERIC precision/scale. Buffer capacity is not numeric precision. + py::str formatSpec("f"); + info.paramCType = SQL_C_CHAR; + info.bufferSize = 1; + for (size_t rowIndex = 0; rowIndex < values.size(); ++rowIndex) { + if (values[rowIndex].is_none()) { + continue; + } + py::object formatted = + steal(PyObject_Format(values[rowIndex].ptr(), formatSpec.ptr())); + if (!formatted) + throw py::error_already_set(); + py::object ascii = steal(PyUnicode_AsASCIIString(formatted.ptr())); + if (!ascii) + throw py::error_already_set(); + info.bufferSize = + std::max(info.bufferSize, static_cast(PyBytes_GET_SIZE(ascii.ptr()))); + detectedValues[rowIndex] = std::move(ascii); + } } - std::vector valueInfos = DetectParamTypes(values.ptr(), Py_None); - columnInfos[columnIndex] = MergeTableColumnInfo(values, valueInfos, columnIndex); + columnValues[columnIndex] = std::move(detectedValues); } return columnInfos; } @@ -641,9 +613,9 @@ static SQLRETURN ReadDescriptorString(SQLHDESC descriptor, SQLSMALLINT recordNum return rc; } -[[noreturn]] static void ThrowTableMetadataError(const SqlHandlePtr& metadataHandle, SQLRETURN rc, - const std::string& operation) { - ErrorInfo error = SQLCheckError_Wrap(SQL_HANDLE_STMT, metadataHandle, rc); +[[noreturn]] static void ThrowTableError(const SqlHandlePtr& statementHandle, SQLRETURN rc, + const std::string& operation) { + ErrorInfo error = SQLCheckError_Wrap(SQL_HANDLE_STMT, statementHandle, rc); py::object raiseException = py::module_::import("mssql_python.exceptions").attr("raise_exception"); raiseException(error.sqlState, operation + ": " + error.ddbcErrorMsg); @@ -684,7 +656,7 @@ static std::vector LoadTableColumnMetadata(const SqlHandlePt reinterpret_cast(static_cast(SQL_TRUE)), SQL_IS_INTEGER); if (!SQL_SUCCEEDED(rc)) { - ThrowTableMetadataError(metadataHandle, rc, "Failed to enable metadata identifier mode"); + ThrowTableError(metadataHandle, rc, "Failed to enable metadata identifier mode"); } TableMetadataScope metadataScope(hMetadataStmt); @@ -693,7 +665,7 @@ static std::vector LoadTableColumnMetadata(const SqlHandlePt reinterpret_cast(static_cast(SQL_SS_NAME_SCOPE_TABLE_TYPE)), SQL_IS_INTEGER); if (!SQL_SUCCEEDED(rc)) { - ThrowTableMetadataError(metadataHandle, rc, "Failed to select table-type metadata scope"); + ThrowTableError(metadataHandle, rc, "Failed to select table-type metadata scope"); } { py::gil_scoped_release release; @@ -705,7 +677,7 @@ static std::vector LoadTableColumnMetadata(const SqlHandlePt SQL_NTS, nullptr, 0); } if (!SQL_SUCCEEDED(rc)) { - ThrowTableMetadataError(metadataHandle, rc, "Failed to discover TVP columns"); + ThrowTableError(metadataHandle, rc, "Failed to discover TVP columns"); } std::vector columns; @@ -718,7 +690,7 @@ static std::vector LoadTableColumnMetadata(const SqlHandlePt break; } if (!SQL_SUCCEEDED(rc)) { - ThrowTableMetadataError(metadataHandle, rc, "Failed to fetch TVP column metadata"); + ThrowTableError(metadataHandle, rc, "Failed to fetch TVP column metadata"); } SQLSMALLINT sqlType = SQL_UNKNOWN_TYPE; @@ -740,7 +712,7 @@ static std::vector LoadTableColumnMetadata(const SqlHandlePt } } if (!SQL_SUCCEEDED(rc)) { - ThrowTableMetadataError(metadataHandle, rc, "Failed to read TVP column metadata"); + ThrowTableError(metadataHandle, rc, "Failed to read TVP column metadata"); } columns.push_back({ @@ -819,10 +791,11 @@ static SQLRETURN ResolveTableValuedParamTypes(SqlHandle& handle, SQLHANDLE hStmt return SQL_SUCCESS; } -static SQLRETURN BindTableValuedParameter(SqlHandle& handle, SQLHANDLE hStmt, int paramIndex, - const py::handle& param, - std::vector>& paramBuffers, - const std::string& charEncoding) { +static SQLRETURN BindTableValuedParameter(const SqlHandlePtr& statementHandle, SQLHANDLE hStmt, + int paramIndex, const py::handle& param, + std::vector>& paramBuffers, + const std::string& charEncoding) { + SqlHandle& handle = *statementHandle; const py::sequence rows = py::reinterpret_borrow(param); const SQLSMALLINT recordNumber = static_cast(paramIndex + 1); @@ -865,27 +838,27 @@ static SQLRETURN BindTableValuedParameter(SqlHandle& handle, SQLHANDLE hStmt, in columnValues[columnIndex].cast().append(row[columnIndex]); } } - columnInfos = DetectTableColumnTypes(columnValues, declaredColumns); + columnInfos = DetectTableColumnTypes(columnValues); for (size_t columnIndex = 0; columnIndex < columnCount; ++columnIndex) { ParamInfo& inferred = columnInfos[columnIndex]; const DescribedParamInfo& declared = declaredColumns[columnIndex]; - inferred.paramSQLType = declared.sqlType; - if (declared.columnSize == 0) { - if (declared.sqlType == SQL_WVARCHAR && inferred.columnSize > MAX_INLINE_CHAR) { + // Bind the source representation; SQL Server converts it to the table's + // declared column type. Only an all-NULL column needs destination metadata. + if (inferred.paramCType == SQL_C_DEFAULT) { + inferred.paramSQLType = declared.sqlType; + inferred.columnSize = std::max(declared.columnSize, 1); + inferred.decimalDigits = declared.decimalDigits; + } else if (declared.columnSize == 0) { + if (inferred.paramSQLType == SQL_WVARCHAR && inferred.columnSize > MAX_INLINE_CHAR) { inferred.paramSQLType = SQL_WLONGVARCHAR; - } else if (declared.sqlType == SQL_VARCHAR && + } else if (inferred.paramSQLType == SQL_VARCHAR && inferred.columnSize > MAX_INLINE_BINARY) { inferred.paramSQLType = SQL_LONGVARCHAR; - } else if (declared.sqlType == SQL_VARBINARY && + } else if (inferred.paramSQLType == SQL_VARBINARY && inferred.columnSize > MAX_INLINE_BINARY) { inferred.paramSQLType = SQL_LONGVARBINARY; } } - inferred.decimalDigits = declared.decimalDigits; - if (inferred.paramCType == SQL_C_DEFAULT || declared.sqlType == SQL_DECIMAL || - declared.sqlType == SQL_NUMERIC) { - inferred.columnSize = std::max(declared.columnSize, 1); - } } } @@ -927,26 +900,28 @@ static SQLRETURN BindTableValuedParameter(SqlHandle& handle, SQLHANDLE hStmt, in return rc; } - SQLRETURN bindRc = SQL_ERROR; try { - bindRc = BindParameterArray(handle, hStmt, columnValues, columnInfos, rows.size(), - paramBuffers, charEncoding); + rc = BindParameterArray(handle, hStmt, columnValues, columnInfos, rows.size(), + paramBuffers, charEncoding); + if (!SQL_SUCCEEDED(rc)) { + // Successful focus restoration clears the original statement diagnostic. + ThrowTableError(statementHandle, rc, "Failed to bind TVP columns"); + } } catch (...) { SQLSetStmtAttr_ptr(hStmt, SQL_SOPT_SS_PARAM_FOCUS, nullptr, SQL_IS_INTEGER); throw; } - const SQLRETURN restoreRc = - SQLSetStmtAttr_ptr(hStmt, SQL_SOPT_SS_PARAM_FOCUS, nullptr, SQL_IS_INTEGER); - return SQL_SUCCEEDED(bindRc) ? restoreRc : bindRc; + return SQLSetStmtAttr_ptr(hStmt, SQL_SOPT_SS_PARAM_FOCUS, nullptr, SQL_IS_INTEGER); } // Given a list of parameters and their ParamInfo, calls SQLBindParameter on // each of them with appropriate arguments -SQLRETURN BindParameters(SqlHandle& handle, SQLHANDLE hStmt, const py::list& params, +SQLRETURN BindParameters(const SqlHandlePtr& statementHandle, SQLHANDLE hStmt, const py::list& params, std::vector& paramInfos, std::vector>& paramBuffers, const std::string& charEncoding = "utf-8") { PERF_TIMER("BindParameters"); + SqlHandle& handle = *statementHandle; LOG("BindParameters: Starting parameter binding for statement handle %p " "with %zu parameters", (void*)hStmt, params.size()); @@ -966,8 +941,8 @@ SQLRETURN BindParameters(SqlHandle& handle, SQLHANDLE hStmt, const py::list& par SQLLEN* strLenOrIndPtr = nullptr; if (paramInfo.isTVP) { - RETCODE rc = BindTableValuedParameter(handle, hStmt, paramIndex, param, paramBuffers, - charEncoding); + RETCODE rc = BindTableValuedParameter(statementHandle, hStmt, paramIndex, param, + paramBuffers, charEncoding); if (!SQL_SUCCEEDED(rc)) { return rc; } @@ -2605,7 +2580,7 @@ SQLRETURN SQLExecute_wrap(const SqlHandlePtr statementHandle, return rc; std::vector> paramBuffers; - rc = BindParameters(*statementHandle, hStmt, params, paramInfos, paramBuffers, charEncoding); + rc = BindParameters(statementHandle, hStmt, params, paramInfos, paramBuffers, charEncoding); if (!SQL_SUCCEEDED(rc)) return rc; { @@ -3419,7 +3394,7 @@ SQLRETURN SQLExecuteMany_wrap(const SqlHandlePtr statementHandle, const std::u16 py::list rowParams = columnwise_params[rowIndex]; std::vector> paramBuffers; - rc = BindParameters(*statementHandle, hStmt, rowParams, paramInfos, + rc = BindParameters(statementHandle, hStmt, rowParams, paramInfos, paramBuffers, charEncoding); if (!SQL_SUCCEEDED(rc)) { LOG("SQLExecuteMany: BindParameters failed for row %zu - rc=%d", rowIndex, rc); diff --git a/tests/test_004_cursor_timeout.py b/tests/test_004_cursor_timeout.py new file mode 100644 index 000000000..d9cc4a8ac --- /dev/null +++ b/tests/test_004_cursor_timeout.py @@ -0,0 +1,75 @@ +"""Regression tests for query-timeout statement-handle routing.""" + +from unittest.mock import Mock + +import pytest + +from mssql_python import ddbc_bindings +from mssql_python.constants import ConstantsDDBC +from mssql_python.cursor import Cursor, logger + + +@pytest.mark.parametrize("metadata_handle", [False, True], ids=["default", "metadata"]) +@pytest.mark.parametrize("timeout", [0, 2], ids=["disabled", "enabled"]) +def test_set_timeout_uses_target_handle(monkeypatch, metadata_handle, timeout): + cursor = Cursor.__new__(Cursor) + cursor.hstmt = object() + cursor._timeout = timeout + statement_handle = object() if metadata_handle else None + target = statement_handle or cursor.hstmt + set_attribute = Mock(return_value=0) + check_error = Mock() + warning = Mock() + monkeypatch.setattr(ddbc_bindings, "DDBCSQLSetStmtAttr", set_attribute) + monkeypatch.setattr("mssql_python.cursor.check_error", check_error) + monkeypatch.setattr(logger, "warning", warning) + + assert cursor._set_timeout(statement_handle) is None + + if timeout: + set_attribute.assert_called_once_with( + target, ConstantsDDBC.SQL_ATTR_QUERY_TIMEOUT.value, timeout + ) + check_error.assert_called_once_with(ConstantsDDBC.SQL_HANDLE_STMT.value, target, 0) + else: + set_attribute.assert_not_called() + check_error.assert_not_called() + warning.assert_not_called() + + +@pytest.mark.parametrize("metadata_handle", [False, True], ids=["default", "metadata"]) +def test_set_timeout_failure_reports_target_diagnostics( + db_connection, monkeypatch, metadata_handle +): + with db_connection.cursor() as cursor: + cursor._timeout = 2 + if metadata_handle: + cursor._tvp_metadata_hstmt = db_connection._conn.alloc_statement_handle() + statement_handle = cursor._tvp_metadata_hstmt + target = statement_handle or cursor.hstmt + real_set_attribute = ddbc_bindings.DDBCSQLSetStmtAttr + real_check_error = ddbc_bindings.DDBCSQLCheckError + + def fail_attribute(handle, attribute, value): + # An invalid identifier produces real driver diagnostics on this handle. + return real_set_attribute(handle, 999999, value) + + set_attribute = Mock(side_effect=fail_attribute) + check_error = Mock(wraps=real_check_error) + warning = Mock() + with monkeypatch.context() as patch: + patch.setattr(ddbc_bindings, "DDBCSQLSetStmtAttr", set_attribute) + patch.setattr(ddbc_bindings, "DDBCSQLCheckError", check_error) + patch.setattr(logger, "warning", warning) + assert cursor._set_timeout(statement_handle) is None + + set_attribute.assert_called_once_with(target, ConstantsDDBC.SQL_ATTR_QUERY_TIMEOUT.value, 2) + check_error.assert_called_once_with(ConstantsDDBC.SQL_HANDLE_STMT.value, target, -1) + assert check_error.call_args.args[1] is target + error_info = real_check_error(ConstantsDDBC.SQL_HANDLE_STMT.value, target, -1) + assert error_info.sqlState == "HY092" + assert error_info.ddbcErrorMsg + warning.assert_called_once() + assert warning.call_args.args[0] == "Failed to set query timeout: %s" + assert "Invalid attribute/option identifier" in warning.call_args.args[1] + assert cursor.execute("SELECT 1").fetchone()[0] == 1 diff --git a/tests/test_028_table_valued_parameters.py b/tests/test_028_table_valued_parameters.py index 4ce1c40a6..4dd5f5d5f 100644 --- a/tests/test_028_table_valued_parameters.py +++ b/tests/test_028_table_valued_parameters.py @@ -11,6 +11,7 @@ from mssql_python import ( DatabaseError, NotSupportedError, + ProgrammingError, SQL_INTEGER, SQL_WVARCHAR, ddbc_bindings, @@ -381,3 +382,214 @@ def record_timeout(statement_handle, attribute, value): assert len(timeout_handles) == 2 assert timeout_handles[0] is not timeout_handles[1] + + +@pytest.fixture +def conversion_tvp(cursor, db_connection, request): + suffix = uuid.uuid4().hex + type_name = f"pytest_conversion_type_{suffix}" + procedure_name = f"pytest_conversion_proc_{suffix}" + try: + cursor.execute( + f"CREATE TYPE dbo.[{type_name}] AS TABLE (position int, value {request.param} NULL)" + ) + cursor.execute( + f"CREATE PROCEDURE dbo.[{procedure_name}] @items dbo.[{type_name}] READONLY AS " + "SET NOCOUNT ON; SELECT value FROM @items ORDER BY position" + ) + db_connection.commit() + yield f"EXEC dbo.[{procedure_name}] ?" + finally: + cursor.execute(f"DROP PROCEDURE IF EXISTS dbo.[{procedure_name}]") + cursor.execute(f"DROP TYPE IF EXISTS dbo.[{type_name}]") + db_connection.commit() + + +@pytest.mark.parametrize( + ("conversion_tvp", "values", "expected"), + [ + ( + "float", + [Decimal("1.5"), Decimal("123.45"), None, Decimal("1.50"), Decimal("-0.125")], + [1.5, 123.45, None, 1.5, -0.125], + ), + ("int", [Decimal("1.5"), Decimal("-123.45"), None], [1, -123, None]), + ("nvarchar(40)", [Decimal("1.5"), None], ["1.5", None]), + ( + "nvarchar(40)", + [Decimal("1.5"), Decimal("123.45"), None, Decimal("-0.125")], + ["1.500", "123.450", None, "-0.125"], + ), + ( + "decimal(5,2)", + [Decimal("1.239"), None, Decimal("-1.235"), Decimal("9.999")], + [Decimal("1.24"), None, Decimal("-1.24"), Decimal("10.00")], + ), + ( + "decimal(8,2)", + ["000000000000000001.23", None, "-0000000000000001.239"], + [Decimal("1.23"), None, Decimal("-1.24")], + ), + ( + "datetime", + [datetime.datetime(2026, 9, 15, 12, 34, 56, 456789), None], + [datetime.datetime(2026, 9, 15, 12, 34, 56, 457000), None], + ), + ( + "datetime2(3)", + [datetime.datetime(2026, 9, 15, 23, 59, 59, 999999), None], + [datetime.datetime(2026, 9, 16), None], + ), + ( + "time(3)", + [datetime.time(12, 34, 56, 456789), None], + [datetime.time(12, 34, 56, 457000), None], + ), + ( + "datetimeoffset(3)", + [ + datetime.datetime(2026, 9, 15, 23, 59, 59, 999999, tzinfo=datetime.timezone.utc), + None, + ], + [datetime.datetime(2026, 9, 16, tzinfo=datetime.timezone.utc), None], + ), + ], + indirect=["conversion_tvp"], +) +def test_tvp_source_precision_survives_destination_conversion( + cursor, conversion_tvp, values, expected +): + rows = list(enumerate(values)) + for _ in range(2): + cursor.execute(conversion_tvp, (rows,)) + assert [row[0] for row in cursor.fetchall()] == expected + assert rows == list(enumerate(values)) + + +@pytest.mark.parametrize("conversion_tvp", ["nvarchar(3)"], indirect=True) +@pytest.mark.parametrize( + ("value", "message", "driver_error"), + [ + ("x" * 5000, "would be truncated", "Syntax error or access violation"), + ("\u0100" * 5000, "Invalid precision value", "Invalid precision or scale value"), + ], + ids=["server-conversion-error", "bind-error"], +) +def test_tvp_error_keeps_diagnostics_and_restores_focus( + cursor, conversion_tvp, value, message, driver_error +): + with pytest.raises(ProgrammingError, match=message) as raised: + cursor.execute(conversion_tvp, ([(0, value)],)) + assert raised.value.driver_error == driver_error + cursor.execute("SELECT ?", 42) + assert cursor.fetchone()[0] == 42 + cursor.execute(conversion_tvp, ([(0, "ok")],)) + assert cursor.fetchone()[0] == "ok" + + +@pytest.mark.parametrize("conversion_tvp", ["decimal(38,0)"], indirect=True) +@pytest.mark.parametrize( + "value", [Decimal("NaN"), Decimal("Infinity"), Decimal("1E+39"), Decimal("1E-39")] +) +def test_tvp_decimal_validation_matches_scalar(cursor, conversion_tvp, value): + with pytest.raises(ValueError): + cursor.execute(conversion_tvp, ([(0, value)],)) + cursor.execute(conversion_tvp, ([(0, Decimal("1E+37")), (1, None)],)) + assert [row[0] for row in cursor.fetchall()] == [Decimal("1E+37"), None] + + +@pytest.mark.parametrize("conversion_tvp", ["bigint"], indirect=True) +def test_tvp_integer_widening_and_nulls(cursor, conversion_tvp): + edges = [0, 255, -32768, 32767, -(2**31), 2**31 - 1, -(2**63), 2**63 - 1] + for ordered in (edges, list(reversed(edges))): + values = [None, *ordered, None] + cursor.execute(conversion_tvp, (list(enumerate(values)),)) + assert [row[0] for row in cursor.fetchall()] == values + + +@pytest.mark.parametrize( + ("conversion_tvp", "values"), + [ + ("nvarchar(max)", [None, "ascii first", "\U0001f600" * 4001, "a\x00b", ""]), + ("nvarchar(max)", ["\U0001f600", "later ascii", None]), + ("varchar(max)", [None, "x" * 8001, "a\x00b", ""]), + ("varbinary(max)", [b"", None, b"\x00\xff" * 4501, bytearray(b"last")]), + ], + indirect=["conversion_tvp"], +) +def test_tvp_source_buffers_cover_every_row(cursor, conversion_tvp, values): + cursor.execute(conversion_tvp, (list(enumerate(values)),)) + assert [row[0] for row in cursor.fetchall()] == values + + +def test_tvp_all_null_columns_use_declared_types(cursor, typed_tvp_procedure): + cursor.execute(f"EXEC dbo.[{typed_tvp_procedure}] ?", ([tuple([None] * 15)],)) + assert tuple(cursor.fetchone()) == tuple([None] * 15) + + +@pytest.mark.parametrize("conversion_tvp", ["float"], indirect=True) +def test_tvp_rejects_unrepresentable_common_decimal_precision(cursor, conversion_tvp): + with pytest.raises(ValueError, match="precision greater than 38"): + cursor.execute(conversion_tvp, ([(0, Decimal("1E+37")), (1, Decimal("0.1"))],)) + cursor.execute(conversion_tvp, ([(0, Decimal("0.1"))],)) + assert cursor.fetchone()[0] == 0.1 + + +@pytest.mark.parametrize("conversion_tvp", ["decimal(5,2)"], indirect=True) +def test_tvp_decimal_wire_format_is_independent_of_text_encoding(conn_str, conversion_tvp): + from mssql_python import SQL_CHAR, connect + + with connect(conn_str) as connection: + connection.setencoding("utf-16le", ctype=SQL_CHAR) + with connection.cursor() as encoded_cursor: + encoded_cursor.execute(conversion_tvp, ([(0, Decimal("-1.239")), (1, None)],)) + assert [row[0] for row in encoded_cursor.fetchall()] == [Decimal("-1.24"), None] + + +@pytest.mark.parametrize("conversion_tvp", ["nvarchar(40)"], indirect=True) +@pytest.mark.parametrize( + ("input_sizes", "error", "message"), + [ + (None, RuntimeError, "metadata requires a valid statement handle"), + ([(SQL_INTEGER, 10, 0)], TypeError, "setinputsizes cannot override"), + ], +) +def test_tvp_native_entry_point_validates_metadata_and_overrides( + db_connection, conversion_tvp, input_sizes, error, message +): + with db_connection.cursor() as native_cursor: + with pytest.raises(error, match=message): + ddbc_bindings.DDBCSQLExecute( + native_cursor.hstmt, + None, + conversion_tvp, + [[(0, "ok")]], + input_sizes, + [False], + True, + {}, + ) + native_cursor.execute(conversion_tvp, ([(0, "ok")],)) + assert native_cursor.fetchone()[0] == "ok" + + +@pytest.mark.parametrize( + ("rows", "error", "message"), + [ + ([1], TypeError, "rows must be list or tuple"), + ([()], ValueError, "at least one column"), + ([(1,)], ValueError, "declared table type requires"), + ([(1, "ok"), 2], TypeError, "rows must be list or tuple"), + ([(1, [2])], TypeError, "nested row sequences"), + ([(1, Decimal("1")), (2, "2")], TypeError, "incompatible Python types"), + ], +) +def test_tvp_shape_and_type_errors_leave_cursor_reusable( + cursor, tvp_procedure, rows, error, message +): + _, procedure_name = tvp_procedure + sql = f"EXEC dbo.[{procedure_name}] ?, ?, ?" + with pytest.raises(error, match=message): + cursor.execute(sql, (0, rows, "")) + cursor.execute(sql, (0, [(1, "ok")], "")) + assert tuple(cursor.fetchone()) == (1, "ok") From d8f0a07df33b53623c483c5fcc2ce4c06bb22629 Mon Sep 17 00:00:00 2001 From: Gaurav Sharma Date: Wed, 16 Sep 2026 00:14:21 +0530 Subject: [PATCH 05/15] FIX: Prevent TVP metadata GIL deadlocks Release the GIL during TVP metadata reads and cleanup so Python forwarding threads can make progress. Preserve shutdown-safe destructor behavior and add bounded regressions for normal execution and metadata-error recovery. AB#48177 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- mssql_python/pybind/ddbc_bindings.cpp | 41 ++++--- tests/test_028_table_valued_parameters.py | 125 ++++++++++++++++++++++ 2 files changed, 152 insertions(+), 14 deletions(-) diff --git a/mssql_python/pybind/ddbc_bindings.cpp b/mssql_python/pybind/ddbc_bindings.cpp index 94af75ef0..fbb4a6985 100644 --- a/mssql_python/pybind/ddbc_bindings.cpp +++ b/mssql_python/pybind/ddbc_bindings.cpp @@ -27,6 +27,7 @@ SQLRETURN BindParameterArray(SqlHandle& handle, SQLHANDLE hStmt, const py::list& std::vector& paramInfos, size_t paramSetSize, std::vector>& paramBuffers, const std::string& charEncoding); +static bool is_python_finalizing(); //------------------------------------------------------------------------------------------------- // Macro definitions @@ -626,7 +627,13 @@ class TableMetadataScope { public: explicit TableMetadataScope(SQLHANDLE statementHandle) : handle(statementHandle) {} ~TableMetadataScope() { - SQLFreeStmt_ptr(handle, SQL_CLOSE); + // Error paths can leave unread metadata; closing it can require network I/O. + if (!is_python_finalizing() && PyGILState_Check()) { + py::gil_scoped_release release; + SQLFreeStmt_ptr(handle, SQL_CLOSE); + } else { + SQLFreeStmt_ptr(handle, SQL_CLOSE); + } SQLSetStmtAttr_ptr( handle, SQL_SOPT_SS_NAME_SCOPE, reinterpret_cast(static_cast(SQL_SS_NAME_SCOPE_TABLE)), @@ -649,8 +656,11 @@ static std::vector LoadTableColumnMetadata(const SqlHandlePt } SQLHANDLE hMetadataStmt = metadataHandle->get(); - SQLFreeStmt_ptr(hMetadataStmt, SQL_CLOSE); - SQLFreeStmt_ptr(hMetadataStmt, SQL_RESET_PARAMS); + { + py::gil_scoped_release release; + SQLFreeStmt_ptr(hMetadataStmt, SQL_CLOSE); + SQLFreeStmt_ptr(hMetadataStmt, SQL_RESET_PARAMS); + } SQLRETURN rc = SQLSetStmtAttr_ptr(hMetadataStmt, SQL_ATTR_METADATA_ID, reinterpret_cast(static_cast(SQL_TRUE)), @@ -698,17 +708,20 @@ static std::vector LoadTableColumnMetadata(const SqlHandlePt SQLSMALLINT decimalDigits = 0; SQLLEN indicator = 0; - rc = SQLGetData_ptr(hMetadataStmt, 5, SQL_C_SSHORT, &sqlType, sizeof(sqlType), &indicator); - if (SQL_SUCCEEDED(rc)) { - rc = SQLGetData_ptr(hMetadataStmt, 7, SQL_C_SLONG, &columnSize, sizeof(columnSize), - &indicator); - } - if (SQL_SUCCEEDED(rc)) { - rc = SQLGetData_ptr(hMetadataStmt, 9, SQL_C_SSHORT, &decimalDigits, - sizeof(decimalDigits), &indicator); - if (indicator == SQL_NULL_DATA) { - decimalDigits = 0; - rc = SQL_SUCCESS; + { + py::gil_scoped_release release; + rc = SQLGetData_ptr(hMetadataStmt, 5, SQL_C_SSHORT, &sqlType, sizeof(sqlType), &indicator); + if (SQL_SUCCEEDED(rc)) { + rc = SQLGetData_ptr(hMetadataStmt, 7, SQL_C_SLONG, &columnSize, sizeof(columnSize), + &indicator); + } + if (SQL_SUCCEEDED(rc)) { + rc = SQLGetData_ptr(hMetadataStmt, 9, SQL_C_SSHORT, &decimalDigits, + sizeof(decimalDigits), &indicator); + if (indicator == SQL_NULL_DATA) { + decimalDigits = 0; + rc = SQL_SUCCESS; + } } } if (!SQL_SUCCEEDED(rc)) { diff --git a/tests/test_028_table_valued_parameters.py b/tests/test_028_table_valued_parameters.py index 4dd5f5d5f..2694ac522 100644 --- a/tests/test_028_table_valued_parameters.py +++ b/tests/test_028_table_valued_parameters.py @@ -3,6 +3,9 @@ """ import datetime +import os +import subprocess +import sys import uuid from decimal import Decimal @@ -17,6 +20,12 @@ ddbc_bindings, ) from mssql_python.odbc_provider import ProviderManager +from test_023_ssh_tunnel_gil_release import ( + WATCHDOG_SECONDS, + _parse_server, + _replace_server, + _start_forwarder, +) @pytest.fixture @@ -593,3 +602,119 @@ def test_tvp_shape_and_type_errors_leave_cursor_reusable( cursor.execute(sql, (0, rows, "")) cursor.execute(sql, (0, [(1, "ok")], "")) assert tuple(cursor.fetchone()) == (1, "ok") + + +@pytest.fixture +def wide_tvp_procedure(cursor, db_connection): + suffix = uuid.uuid4().hex + type_name = f"pytest_wide_tvp_type_{suffix}" + procedure_name = f"pytest_wide_tvp_proc_{suffix}" + # Long column names make metadata span multiple network packets. + columns = ", ".join(["id int"] + [f"c{i}_{'x' * 100} int" for i in range(1, 1024)]) + try: + cursor.execute(f"CREATE TYPE dbo.[{type_name}] AS TABLE ({columns})") + cursor.execute( + f"CREATE PROCEDURE dbo.[{procedure_name}] @items dbo.[{type_name}] READONLY AS " + "SET NOCOUNT ON; SELECT id FROM @items ORDER BY id" + ) + db_connection.commit() + yield procedure_name + finally: + cursor.execute(f"DROP PROCEDURE IF EXISTS dbo.[{procedure_name}]") + cursor.execute(f"DROP TYPE IF EXISTS dbo.[{type_name}]") + db_connection.commit() + + +def _run_forwarded_tvp(): + import mssql_python + + base = os.environ["DB_CONNECTION_STRING"] + target = _parse_server(base) + assert target is not None, "Could not parse Server=host,port" + host, port = _start_forwarder(target) + mssql_python.pooling(enabled=False) + with mssql_python.connect(_replace_server(base, host, port)) as connection: + with connection.cursor() as cursor: + sql = f"EXEC dbo.[{os.environ['TVP_GIL_PROCEDURE']}] ?" + padding = (None,) * 1023 + if os.environ["TVP_GIL_METADATA_ERROR"] == "1": + import ctypes + + library = ctypes.CDLL(sys.modules["ddbc_bindings"].__file__) + slot = ctypes.c_void_p.in_dll(library, "SQLGetData_ptr") + original = slot.value + get_data_type = ctypes.CFUNCTYPE( + ctypes.c_short, + ctypes.c_void_p, + ctypes.c_ushort, + ctypes.c_short, + ctypes.c_void_p, + ctypes.c_ssize_t, + ctypes.POINTER(ctypes.c_ssize_t), + ) + get_data = get_data_type(original) + + @get_data_type + def invalid_metadata_column(handle, column, ctype, value, size, indicator): + # Produce a real driver error before metadata is drained. Do not + # wrap SQLFreeStmt: its original GIL behavior is what we exercise. + slot.value = original + return get_data(handle, 999, ctype, value, size, indicator) + + slot.value = ctypes.cast(invalid_metadata_column, ctypes.c_void_p).value + try: + with pytest.raises(ProgrammingError, match="(?i)invalid descriptor index"): + cursor.execute(sql, ([(1,) + padding],)) + finally: + slot.value = original + cursor.execute("SELECT ?", 42) + assert [tuple(row) for row in cursor.fetchall()] == [(42,)] + + # No pointer overrides on the normal path, including after error recovery. + for ids in ([1, 2], [3], []): + cursor.execute(sql, ([(i,) + padding for i in ids],)) + assert [tuple(row) for row in cursor.fetchall()] == [(i,) for i in ids] + print("OK forwarded TVP metadata", flush=True) + + +@pytest.mark.parametrize( + "metadata_error", + [ + pytest.param(False, id="metadata-reads"), + pytest.param( + True, + id="metadata-error-cleanup", + marks=pytest.mark.skipif( + sys.platform == "win32", reason="Native function-pointer injection is Unix-only" + ), + ), + ], +) +def test_tvp_metadata_through_python_forwarder_does_not_deadlock( + conn_str, wide_tvp_procedure, metadata_error +): + if not conn_str or _parse_server(conn_str) is None: + pytest.skip("Requires DB_CONNECTION_STRING with Server=host,port") + env = os.environ.copy() + env["PYTHONPATH"] = os.pathsep.join(sys.path) + env["TVP_GIL_PROCEDURE"] = wide_tvp_procedure + env["TVP_GIL_METADATA_ERROR"] = str(int(metadata_error)) + try: + result = subprocess.run( + [ + sys.executable, + "-c", + "from test_028_table_valued_parameters import _run_forwarded_tvp; " + "_run_forwarded_tvp()", + ], + env=env, + capture_output=True, + text=True, + timeout=WATCHDOG_SECONDS, + ) + except subprocess.TimeoutExpired: + pytest.fail( + f"TVP metadata through the Python forwarder deadlocked after {WATCHDOG_SECONDS}s" + ) + assert result.returncode == 0, result.stdout + result.stderr + assert "OK forwarded TVP metadata" in result.stdout From 55be66b4a50e4e8bd6475d379fc1792c0b221746 Mon Sep 17 00:00:00 2001 From: Gaurav Sharma Date: Wed, 16 Sep 2026 00:52:23 +0530 Subject: [PATCH 06/15] FIX: Promote long TVP sources for bounded destinations Choose long TVP SQL types from source length regardless of destination size. Preserve valid Unicode values in bounded varchar columns and let SQL Server enforce destination limits. Add boundary and cursor-recovery regressions. AB#48177 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- mssql_python/pybind/ddbc_bindings.cpp | 2 +- tests/test_028_table_valued_parameters.py | 36 +++++++++++++++-------- 2 files changed, 25 insertions(+), 13 deletions(-) diff --git a/mssql_python/pybind/ddbc_bindings.cpp b/mssql_python/pybind/ddbc_bindings.cpp index fbb4a6985..df0bcf9b5 100644 --- a/mssql_python/pybind/ddbc_bindings.cpp +++ b/mssql_python/pybind/ddbc_bindings.cpp @@ -861,7 +861,7 @@ static SQLRETURN BindTableValuedParameter(const SqlHandlePtr& statementHandle, S inferred.paramSQLType = declared.sqlType; inferred.columnSize = std::max(declared.columnSize, 1); inferred.decimalDigits = declared.decimalDigits; - } else if (declared.columnSize == 0) { + } else { if (inferred.paramSQLType == SQL_WVARCHAR && inferred.columnSize > MAX_INLINE_CHAR) { inferred.paramSQLType = SQL_WLONGVARCHAR; } else if (inferred.paramSQLType == SQL_VARCHAR && diff --git a/tests/test_028_table_valued_parameters.py b/tests/test_028_table_valued_parameters.py index 2694ac522..154de1022 100644 --- a/tests/test_028_table_valued_parameters.py +++ b/tests/test_028_table_valued_parameters.py @@ -475,25 +475,26 @@ def test_tvp_source_precision_survives_destination_conversion( assert rows == list(enumerate(values)) -@pytest.mark.parametrize("conversion_tvp", ["nvarchar(3)"], indirect=True) @pytest.mark.parametrize( - ("value", "message", "driver_error"), + ("conversion_tvp", "value", "recovery"), [ - ("x" * 5000, "would be truncated", "Syntax error or access violation"), - ("\u0100" * 5000, "Invalid precision value", "Invalid precision or scale value"), + ("nvarchar(3)", "x" * 5000, "ok"), + ("nvarchar(3)", "\u0100" * 5000, "ok"), + ("varchar(8000)", "x" * 8001, "ok"), + ("varchar(8000) COLLATE Latin1_General_100_CI_AS", "\u00e9" * 8001, "ok"), + ("varbinary(8000)", b"x" * 8001, b"ok"), ], - ids=["server-conversion-error", "bind-error"], + ids=["short-ascii", "short-unicode", "long-ascii", "long-unicode", "long-binary"], + indirect=["conversion_tvp"], ) -def test_tvp_error_keeps_diagnostics_and_restores_focus( - cursor, conversion_tvp, value, message, driver_error -): - with pytest.raises(ProgrammingError, match=message) as raised: +def test_tvp_destination_limits_leave_cursor_reusable(cursor, conversion_tvp, value, recovery): + with pytest.raises(ProgrammingError, match="would be truncated") as raised: cursor.execute(conversion_tvp, ([(0, value)],)) - assert raised.value.driver_error == driver_error + assert raised.value.driver_error == "Syntax error or access violation" cursor.execute("SELECT ?", 42) assert cursor.fetchone()[0] == 42 - cursor.execute(conversion_tvp, ([(0, "ok")],)) - assert cursor.fetchone()[0] == "ok" + cursor.execute(conversion_tvp, ([(0, recovery)],)) + assert cursor.fetchone()[0] == recovery @pytest.mark.parametrize("conversion_tvp", ["decimal(38,0)"], indirect=True) @@ -531,6 +532,17 @@ def test_tvp_source_buffers_cover_every_row(cursor, conversion_tvp, values): assert [row[0] for row in cursor.fetchall()] == values +@pytest.mark.parametrize( + "conversion_tvp", ["varchar(8000) COLLATE Latin1_General_100_CI_AS"], indirect=True +) +@pytest.mark.parametrize("length", [4000, 4001, 8000]) +def test_tvp_unicode_source_fits_bounded_varchar(cursor, conversion_tvp, length): + values = [None, "ascii first", "\u00e9" * length, ""] + for ordered in (values, list(reversed(values))): + cursor.execute(conversion_tvp, (list(enumerate(ordered)),)) + assert [row[0] for row in cursor.fetchall()] == ordered + + def test_tvp_all_null_columns_use_declared_types(cursor, typed_tvp_procedure): cursor.execute(f"EXEC dbo.[{typed_tvp_procedure}] ?", ([tuple([None] * 15)],)) assert tuple(cursor.fetchone()) == tuple([None] * 15) From f83fbab55c55465acb57bcf3806b60b97a42c2e7 Mon Sep 17 00:00:00 2001 From: Gaurav Sharma Date: Tue, 13 Oct 2026 12:56:11 +0530 Subject: [PATCH 07/15] FEAT: Stream explicitly typed table-valued parameters Replace implicit nested-row discovery with one public TableValuedParameter carrying type identity, ordered columns, and rows. Stream synchronous inputs through the existing ODBC data-at-execution loop with one-row binding storage, preserve cancellation and generator errors, and adapt materialized inputs to the existing async native constructor without claiming unsupported streaming. Keep scalar conversions shared, support the full SQL DATE range, and add runtime conformance coverage for both execution paths and incremental ODBC delivery. AB#48177 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- CHANGELOG.md | 10 +- README.md | 62 ++ mssql_python/__init__.py | 2 + mssql_python/async_query/async_execute.py | 40 +- mssql_python/cursor.py | 75 +- mssql_python/mssql_python.pyi | 31 + mssql_python/pybind/ddbc_bindings.cpp | 656 ++---------- mssql_python/pybind/ddbc_bindings.h | 26 +- mssql_python/pybind/param_detect.hpp | 28 +- mssql_python/pybind/tvp.hpp | 172 +++ mssql_python/table_valued_parameter.py | 172 +++ tests/test_004_cursor.py | 3 +- tests/test_004_cursor_timeout.py | 80 +- tests/test_028_table_valued_parameters.py | 1184 ++++++++++----------- 14 files changed, 1233 insertions(+), 1308 deletions(-) create mode 100644 mssql_python/pybind/tvp.hpp create mode 100644 mssql_python/table_valued_parameter.py diff --git a/CHANGELOG.md b/CHANGELOG.md index f4f43ece6..27b946cfa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,10 +7,12 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ## [Unreleased] ### Added -- **AB#48177:** Added input table-valued parameter support for stored procedure - calls. Pass a materialized list or tuple of rows as one `execute()` parameter, - including empty tables and positional or named parameters. The Rust ODBC - provider does not yet support TVPs. +- **AB#48177:** Added explicit `TableValuedParameter(type_name, columns, rows, + schema=...)` input values. Sync `execute()` streams lists, tuples and iterators + through native ODBC data-at-execution using one row of additional storage and + one procedure invocation, without discovering table metadata. Async execution + adapts materialized rows to the pinned native core; iterator streaming is not + available asynchronously. See README for current native async schema limits. - New feature: Support for macOS and Linux. - Documentation: Added API documentation in the Wiki. - New `token_provider=` parameter on `connect()` / `Connection` for Microsoft diff --git a/README.md b/README.md index 40fc409e7..9a879beee 100644 --- a/README.md +++ b/README.md @@ -208,6 +208,68 @@ connection.close() ``` +### Table-valued parameters + +Declare a table type and an input `READONLY` parameter on SQL Server: + +```sql +CREATE TYPE dbo.ItemBatch AS TABLE (id int, label nvarchar(100)); +GO +CREATE PROCEDURE dbo.ProcessItems @items dbo.ItemBatch READONLY +AS SELECT id, label FROM @items; +``` + +Pass one explicit value to `execute()`; ordinary nested lists/tuples are **not** +implicitly treated as TVPs: + +```python +from mssql_python import TableValuedParameter, SQL_INTEGER, SQL_WVARCHAR + +items = TableValuedParameter( + type_name="dbo.ItemBatch", + columns=[("id", SQL_INTEGER), ("label", SQL_WVARCHAR, 100)], + rows=[(1, "Keyboard"), (2, None)], +) +cursor.execute("EXEC dbo.ProcessItems ?", items) +``` + +Column tuples are `(name, sql_type[, size_or_precision[, scale]])`. Use the +driver's integer `SQL_*` constants. Columns bind **positionally**; names are +diagnostic labels, not a schema lookup. Character and binary columns require a +size; `0` means MAX for variable-length types. Decimal columns require both +precision and scale, e.g. `("amount", SQL_DECIMAL, 20, 6)`. Precision is 1–38; +scale is 0–precision. Temporal scale defaults to 7 (0–7 supported synchronously). +Sizes for Unicode columns are UTF-16 code units; binary sizes are bytes. +Rows must contain one cell per column. Supported cells include integers, floats, +bools, `Decimal`, strings, bytes/bytearray, date/time/datetime, UUID and `None`. + +Use an unquoted `schema.Type` or `type_name="ItemBatch", schema="dbo"` (default +schema: `dbo`); SQL quoting/bracket syntax and database-qualified names are not +parsed. The constructor does not consume rows. `rows=None` is invalid: use `[]` +for an empty table; individual cells may be `None`. + +With the default **msodbcsql18** sync provider, `rows` can also be a synchronous +iterator/generator. It is pulled during `execute()`, with one row of additional +driver-side binding storage regardless of total input length. A large cell still +occupies memory for that cell. This is one SQL execution, **not** `executemany()`; +multiple TVPs, scalar parameters and named parameters can be combined. The +native row-count descriptor permits up to 2,147,483,647 rows. Materialized sources +can be reused; iterators are one-shot and may be partially consumed on failure. +Iterator exceptions, including `KeyboardInterrupt`, cancel/reset the statement +before releasing its buffers. `executemany()` and `setinputsizes()` overrides +for statements containing TVPs are not supported. The `mssql-odbc` provider does +not currently support this ODBC TVP path. + +The **same Python value** is accepted by preview async `execute()`, with +materialized list/tuple rows only. The pinned `mssql-python-rs` 0.3.0 constructor +materializes its native table; generators and async generators raise +`NotSupportedError` **without being consumed**, not a hidden `list()` conversion. +Its current TVP metadata builder additionally requires **DECIMAL/NUMERIC(18,10)** +and **temporal scale 7**, even when every cell is non-NULL. Other declarations +propagate the native error. Native streaming and arbitrary decimal/temporal +metadata require follow-up in `mssql-rs`; this driver does not implement or +promise those Rust features. + ## Still have questions? Check out our [FAQ](https://github.com/microsoft/mssql-python/wiki/Frequently-Asked-Questions). Still not answered? Create an [issue](https://github.com/microsoft/mssql-python/issues/new/choose) to ask a question. diff --git a/mssql_python/__init__.py b/mssql_python/__init__.py index 06b382c9e..7710cf56c 100644 --- a/mssql_python/__init__.py +++ b/mssql_python/__init__.py @@ -66,6 +66,7 @@ # Row Objects from .row import Row, RowMapping +from .table_valued_parameter import TableValuedParameter # Logging Configuration (Simplified single-level DEBUG system) from .logging import logger, setup_logging, driver_logger @@ -324,6 +325,7 @@ def _cleanup_connections(): ) __all__ = [ + "TableValuedParameter", # Exception classes "Warning", "Error", diff --git a/mssql_python/async_query/async_execute.py b/mssql_python/async_query/async_execute.py index 6f9bb7b05..8c5f8b4a9 100644 --- a/mssql_python/async_query/async_execute.py +++ b/mssql_python/async_query/async_execute.py @@ -4,7 +4,9 @@ from collections.abc import Mapping, Sequence from typing import TYPE_CHECKING, Any -from ..exceptions import OperationalError, ProgrammingError +from ..exceptions import NotSupportedError, OperationalError, ProgrammingError +from ..table_valued_parameter import TableValuedParameter +from ._native import load_py_core from ..logging import logger from ..row import Row from .exception_translator import translate_py_core_exceptions @@ -50,6 +52,42 @@ async def execute( if len(parameters) == 1 and isinstance(parameters[0], (tuple, list, Row)): parameters = tuple(parameters[0]) + def adapt(value): + if not isinstance(value, TableValuedParameter): + return value + if not isinstance(value.rows, (list, tuple)): + raise NotSupportedError( + "Async TVPs currently require materialized list/tuple rows", + "Use synchronous execute for streaming; native async TVP streaming is not available", + ) + for index, row in enumerate(value.rows): + value._validate_row(row, index) + return load_py_core().TableValuedParameter( + value.type_name, + # py-core selects MAX above the inline width, whereas ODBC uses 0. + columns=[ + ( + {-1: 12, -10: -9, -4: -3}.get(sql_type, sql_type), + 2147483647 if size == 0 and sql_type in (12, -9, -3, -1, -10, -4) else size, + scale, + ) + for _, sql_type, size, scale in value.columns + ], + rows=value.rows, + schema=value.schema, + ) + + # Named parameter dictionaries are passed intact to py-core, just like + # ordinary execute; adapt only explicit TVP values. + parameters = tuple( + ( + {key: adapt(value) for key, value in parameter.items()} + if isinstance(parameter, Mapping) + else adapt(parameter) + ) + for parameter in parameters + ) + logger.debug( "AsyncCursor.execute: starting; param_count=%d; use_prepare=%s; reset_cursor=%s", len(parameters), diff --git a/mssql_python/cursor.py b/mssql_python/cursor.py index 63aaa3b6f..abefae2c3 100644 --- a/mssql_python/cursor.py +++ b/mssql_python/cursor.py @@ -16,6 +16,7 @@ import uuid import datetime import warnings +from collections.abc import AsyncIterable from typing import List, Mapping, Union, Any, Optional, Tuple, Sequence, TYPE_CHECKING, Iterable from mssql_python.constants import ConstantsDDBC as ddbc_sql_const, SQLTypes from mssql_python.helpers import check_error, connstr_to_pycore_params @@ -29,6 +30,7 @@ DatabaseError, ) from mssql_python.row import Row +from mssql_python.table_valued_parameter import TableValuedParameter from mssql_python.perf_timer import perf_phase from mssql_python import get_settings from mssql_python.odbc_provider import ProviderManager, PROVIDER_MSSQL_ODBC @@ -387,10 +389,10 @@ def __init__(self, connection: "Connection", timeout: int = 0) -> None: # an HSTMT exists. self.closed: bool = False self.hstmt: Optional[Any] = None - self._tvp_metadata_hstmt: Optional[Any] = None self._connection: "Connection" = connection # Store as private attribute self._timeout: int = timeout + self._executing = False self._inputsizes: Optional[List[Tuple[int, int, int, int]]] = None # self.connection.autocommit = False self._initialize_cursor() @@ -1036,7 +1038,7 @@ def _allocate_statement_handle(self) -> None: """ self.hstmt = self._connection._conn.alloc_statement_handle() - def _set_timeout(self, statement_handle=None) -> None: + def _set_timeout(self) -> None: """ Set the query timeout attribute on the statement handle. This is called once when the cursor is created and after any handle reallocation. @@ -1046,13 +1048,12 @@ def _set_timeout(self, statement_handle=None) -> None: logger.debug("_set_timeout: Setting query timeout=%d seconds", self._timeout) try: timeout_value = int(self._timeout) - target_handle = statement_handle or self.hstmt ret = ddbc_bindings.DDBCSQLSetStmtAttr( - target_handle, + self.hstmt, ddbc_sql_const.SQL_ATTR_QUERY_TIMEOUT.value, timeout_value, ) - check_error(ddbc_sql_const.SQL_HANDLE_STMT.value, target_handle, ret) + check_error(ddbc_sql_const.SQL_HANDLE_STMT.value, self.hstmt, ret) logger.debug("Query timeout set to %d seconds", timeout_value) except Exception as e: # pylint: disable=broad-exception-caught logger.warning("Failed to set query timeout: %s", str(e)) @@ -1107,6 +1108,8 @@ def close(self) -> None: if getattr(self, "closed", True): # Do nothing - not calling _check_closed() here since we want this to be idempotent return + if getattr(self, "_executing", False): + raise ProgrammingError("Cannot close a cursor while it is executing", "") # Clear messages per DBAPI self.messages = [] @@ -1122,9 +1125,6 @@ def close(self) -> None: self.hstmt.free() self.hstmt = None logger.debug("SQLFreeHandle succeeded") - if self._tvp_metadata_hstmt: - self._tvp_metadata_hstmt.free() - self._tvp_metadata_hstmt = None self._clear_rownumber() self.closed = True @@ -1135,6 +1135,8 @@ def _check_closed(self) -> None: Raises: ProgrammingError: If the cursor is closed. """ + if getattr(self, "_executing", False): + raise ProgrammingError("Cannot reuse a cursor while it is executing", "") if self.closed: raise ProgrammingError( driver_error="Operation cannot be performed: The cursor is closed.", @@ -1805,9 +1807,8 @@ def execute( # pylint: disable=too-many-locals,too-many-branches,too-many-state # # Important: If you pass a tuple/list/dict as the ONLY argument, # it will be unwrapped for parameter binding. This means you cannot - # pass a tuple as a single scalar parameter value. A TVP remains one - # parameter by wrapping its rows in the ordinary outer parameter tuple: - # execute("EXEC dbo.proc ?", (rows,)) + # pass a tuple as a single scalar parameter value. A TVP is explicitly + # wrapped in TableValuedParameter and remains one parameter. with perf_phase("py::execute::param_prep"): if parameters: # Check if single parameter is a nested container that should be unwrapped @@ -1843,9 +1844,19 @@ def execute( # pylint: disable=too-many-locals,too-many-branches,too-many-state # Getting encoding setting encoding_settings = self._get_encoding_settings() + if any( + isinstance(parameter, TableValuedParameter) + and isinstance(parameter.rows, AsyncIterable) + for parameter in parameters + ): + raise NotSupportedError( + "Synchronous TVPs require a synchronous row source", + "Use a list, tuple, or synchronous iterator", + ) + # Validate that inputsizes matches parameter count if both are present if parameters and self._inputsizes: - if any(isinstance(parameter, (list, tuple)) for parameter in parameters): + if any(isinstance(parameter, TableValuedParameter) for parameter in parameters): raise NotSupportedError( "setinputsizes does not support table-valued parameters", "Remove setinputsizes for statements that pass TVP rows", @@ -1867,24 +1878,19 @@ def execute( # pylint: disable=too-many-locals,too-many-branches,too-many-state self.is_stmt_prepared = [False] effective_use_prepare = use_prepare and not same_sql + self._executing = True try: with perf_phase("py::execute::cpp_call"): if parameters: - if any(isinstance(parameter, (list, tuple)) for parameter in parameters): + if any(isinstance(parameter, TableValuedParameter) for parameter in parameters): if ProviderManager.effective() == PROVIDER_MSSQL_ODBC: raise NotSupportedError( "Table-valued parameters are not supported by " "the mssql-odbc provider", "Use the default msodbcsql18 provider for TVP statements", ) - if self._tvp_metadata_hstmt is None: - self._tvp_metadata_hstmt = ( - self._connection._conn.alloc_statement_handle() - ) - self._set_timeout(self._tvp_metadata_hstmt) ret = ddbc_bindings.DDBCSQLExecute( self.hstmt, - self._tvp_metadata_hstmt, operation, parameters, self._inputsizes, @@ -1896,10 +1902,17 @@ def execute( # pylint: disable=too-many-locals,too-many-branches,too-many-state ret = ddbc_bindings.DDBCSQLExecDirect(self.hstmt, operation) # Check for errors but don't raise exceptions for info/warning messages check_error(ddbc_sql_const.SQL_HANDLE_STMT.value, self.hstmt, ret) - except Exception as e: # pylint: disable=broad-exception-caught + except BaseException as e: # Reset even when a TVP iterator raises KeyboardInterrupt. logger.warning("Execute failed, resetting cursor: %s", e) - self._reset_cursor() + try: + self._reset_cursor() + except Exception as cleanup_error: + # A callback may have closed the connection. Preserve its + # original error instead of replacing it with an allocation error. + logger.warning("Cursor recovery failed: %s", cleanup_error) raise + finally: + self._executing = False # Capture any diagnostic messages (SQL_SUCCESS_WITH_INFO, etc.) with perf_phase("py::execute::diag_records"): @@ -2507,6 +2520,7 @@ def executemany( # pylint: disable=too-many-locals,too-many-branches,too-many-s if hasattr(seq_of_parameters, "__getitem__") else next(iter(seq_of_parameters)) ) + if isinstance(first_row, dict): # pyformat style - convert all rows # Parse parameter names from SQL (determines order for all rows) @@ -2571,15 +2585,16 @@ def executemany( # pylint: disable=too-many-locals,too-many-branches,too-many-s # Prepare parameter type information with perf_phase("py::executemany::param_type_detection"): for col_index in range(param_count): - column = [] - for row in seq_of_parameters: - value = row[col_index] - if isinstance(value, (list, tuple)): - raise NotSupportedError( - "executemany does not support table-valued parameters", - "Use execute for each TVP operation", - ) - column.append(value) + column = ( + [row[col_index] for row in seq_of_parameters] + if hasattr(seq_of_parameters, "__getitem__") + else [] + ) + if any(isinstance(value, TableValuedParameter) for value in column): + raise NotSupportedError( + "executemany does not support table-valued parameters", + "Use execute for each TVP operation", + ) sample_value, min_val, max_val, _ = self._compute_column_type(column) if self._inputsizes and col_index < len(self._inputsizes): diff --git a/mssql_python/mssql_python.pyi b/mssql_python/mssql_python.pyi index 18f70c28c..3215ce7b5 100644 --- a/mssql_python/mssql_python.pyi +++ b/mssql_python/mssql_python.pyi @@ -16,11 +16,42 @@ from typing import ( Callable, Iterator, Iterable, + AsyncIterable, ) import datetime import logging import pyarrow +class TableValuedParameter: + def __init__( + self, + type_name: str, + columns: Sequence[Union[Tuple[str, int], Tuple[str, int, int], Tuple[str, int, int, int]]], + rows: Union[ + List[Sequence[Any]], + Tuple[Sequence[Any], ...], + Iterator[Sequence[Any]], + AsyncIterable[Sequence[Any]], + ], + *, + schema: Optional[str] = None, + ) -> None: ... + @property + def type_name(self) -> str: ... + @property + def schema(self) -> str: ... + @property + def columns(self) -> Tuple[Tuple[str, int, int, int], ...]: ... + @property + def rows( + self, + ) -> Union[ + List[Sequence[Any]], + Tuple[Sequence[Any], ...], + Iterator[Sequence[Any]], + AsyncIterable[Sequence[Any]], + ]: ... + # GLOBALS - DB-API 2.0 Required Module Globals # https://www.python.org/dev/peps/pep-0249/#module-interface apilevel: str # "2.0" diff --git a/mssql_python/pybind/ddbc_bindings.cpp b/mssql_python/pybind/ddbc_bindings.cpp index 4b7e0c6b9..0e7630c21 100644 --- a/mssql_python/pybind/ddbc_bindings.cpp +++ b/mssql_python/pybind/ddbc_bindings.cpp @@ -26,11 +26,6 @@ #include // std::forward #include // CPython datetime API (PyDateTime_IMPORT, PyDateTime_GET_*, etc.) -SQLRETURN BindParameterArray(SqlHandle& handle, SQLHANDLE hStmt, const py::list& columnwise_params, - std::vector& paramInfos, size_t paramSetSize, - std::vector>& paramBuffers, - const std::string& charEncoding); -static bool is_python_finalizing(); //------------------------------------------------------------------------------------------------- // Macro definitions @@ -235,7 +230,6 @@ SQLExecuteFunc SQLExecute_ptr = nullptr; SQLRowCountFunc SQLRowCount_ptr = nullptr; SQLGetStmtAttrFunc SQLGetStmtAttr_ptr = nullptr; SQLSetDescFieldFunc SQLSetDescField_ptr = nullptr; -SQLGetDescFieldFunc SQLGetDescField_ptr = nullptr; // Data retrieval APIs SQLFetchFunc SQLFetch_ptr = nullptr; @@ -354,6 +348,8 @@ template // The callable hides whether the caller wraps SQLPutData with GIL management; chunk sizing stays shared. static SQLRETURN stream_dae_chunks(const void* data, size_t total_bytes, PutDataFn put_data_fn) { const char* bytes = static_cast(data); + if (total_bytes == 0) + return put_data_fn(const_cast(data), 0); for (size_t offset = 0; offset < total_bytes; offset += DAE_CHUNK_SIZE) { size_t len = std::min(static_cast(DAE_CHUNK_SIZE), total_bytes - offset); SQLRETURN rc = put_data_fn( @@ -458,167 +454,7 @@ static void PreResolveUnknownNullTypes(SqlHandle& handle, SQLHANDLE hStmt, } } -static bool IsIntegerCType(SQLSMALLINT cType) { - return cType == SQL_C_TINYINT || cType == SQL_C_UTINYINT || cType == SQL_C_SHORT || - cType == SQL_C_LONG || cType == SQL_C_SBIGINT; -} - -static bool IsTextSqlType(SQLSMALLINT sqlType) { - return sqlType == SQL_VARCHAR || sqlType == SQL_WVARCHAR; -} - -static ParamInfo MergeTableColumnInfo(const py::list& values, - const std::vector& valueInfos, - size_t columnIndex) { - ParamInfo merged; - bool hasValue = false; - bool integerColumn = false; - bool textColumn = false; - SQLULEN maxSize = 0; - SQLULEN maxIntegerDigits = 0; - SQLSMALLINT maxScale = 0; - - for (size_t rowIndex = 0; rowIndex < valueInfos.size(); ++rowIndex) { - const ParamInfo& current = valueInfos[rowIndex]; - if (values[rowIndex].is_none()) { - continue; - } - if (current.isTVP) { - throw py::type_error("TVP cells cannot contain nested row sequences"); - } - - const bool currentInteger = IsIntegerCType(current.paramCType); - const bool currentText = - current.paramCType == SQL_C_WCHAR && IsTextSqlType(current.paramSQLType); - if (!hasValue) { - merged = current; - integerColumn = currentInteger; - textColumn = currentText; - hasValue = true; - } else if (integerColumn && currentInteger) { - // Scalar detection already range-checks integers; retain its widest type. - if (current.columnSize > merged.columnSize) { - merged = current; - } - } else if (!(textColumn && currentText) && - (merged.paramCType != current.paramCType || - merged.paramSQLType != current.paramSQLType)) { - throw py::type_error("TVP column " + std::to_string(columnIndex) + - " contains incompatible Python types at row " + - std::to_string(rowIndex)); - } - - if (currentInteger) { - continue; - } else if (currentText) { - if (current.paramSQLType == SQL_WVARCHAR) { - merged.paramSQLType = SQL_WVARCHAR; - } - SQLULEN size = - current.isDAE ? static_cast(current.utf16Len) : current.columnSize; - maxSize = std::max(maxSize, size); - } else if (current.paramCType == SQL_C_BINARY) { - Py_ssize_t size = PyObject_Length(values[rowIndex].ptr()); - if (size < 0) - throw py::error_already_set(); - maxSize = std::max(maxSize, static_cast(size)); - } else if (current.paramCType == SQL_C_NUMERIC) { - SQLULEN integerDigits = - current.columnSize > static_cast(current.decimalDigits) - ? current.columnSize - static_cast(current.decimalDigits) - : 0; - maxIntegerDigits = std::max(maxIntegerDigits, integerDigits); - maxScale = std::max(maxScale, current.decimalDigits); - } else { - maxSize = std::max(maxSize, current.columnSize); - maxScale = std::max(maxScale, current.decimalDigits); - } - } - - if (!hasValue) { - merged.paramCType = SQL_C_DEFAULT; - merged.paramSQLType = SQL_VARCHAR; - merged.columnSize = 1; - merged.decimalDigits = 0; - return merged; - } - - if (integerColumn) { - if (merged.paramCType == SQL_C_TINYINT) { - merged.paramCType = SQL_C_UTINYINT; - } - } else if (textColumn || merged.paramCType == SQL_C_BINARY) { - merged.columnSize = std::max(maxSize, 1); - merged.isDAE = false; - } else if (merged.paramCType == SQL_C_NUMERIC) { - merged.decimalDigits = maxScale; - merged.columnSize = std::max(maxIntegerDigits + maxScale, 1); - } else { - merged.columnSize = std::max(maxSize, 1); - merged.decimalDigits = maxScale; - } - - return merged; -} - -static std::vector DetectTableColumnTypes(py::list& columnValues) { - std::vector columnInfos(columnValues.size()); - for (size_t columnIndex = 0; columnIndex < columnValues.size(); ++columnIndex) { - py::list values = columnValues[columnIndex].cast(); - py::list detectedValues = values.attr("copy")(); - std::vector valueInfos = DetectParamTypes(detectedValues.ptr(), Py_None); - ParamInfo& info = columnInfos[columnIndex]; - info = MergeTableColumnInfo(detectedValues, valueInfos, columnIndex); - if (info.paramCType == SQL_C_NUMERIC) { - if (info.columnSize > MAX_NUMERIC_PRECISION) { - throw py::value_error("TVP Decimal column requires precision greater than 38"); - } - // Preserve each coefficient's scale using exact text, while binding a common - // source NUMERIC precision/scale. Buffer capacity is not numeric precision. - py::str formatSpec("f"); - info.paramCType = SQL_C_CHAR; - info.bufferSize = 1; - for (size_t rowIndex = 0; rowIndex < values.size(); ++rowIndex) { - if (values[rowIndex].is_none()) { - continue; - } - py::object formatted = - steal(PyObject_Format(values[rowIndex].ptr(), formatSpec.ptr())); - if (!formatted) - throw py::error_already_set(); - py::object ascii = steal(PyUnicode_AsASCIIString(formatted.ptr())); - if (!ascii) - throw py::error_already_set(); - info.bufferSize = - std::max(info.bufferSize, static_cast(PyBytes_GET_SIZE(ascii.ptr()))); - detectedValues[rowIndex] = std::move(ascii); - } - } - columnValues[columnIndex] = std::move(detectedValues); - } - return columnInfos; -} - -static SQLRETURN ReadDescriptorString(SQLHDESC descriptor, SQLSMALLINT recordNumber, - SQLSMALLINT fieldIdentifier, std::u16string& value) { - SQLWCHAR buffer[256] = {}; - SQLINTEGER length = 0; - SQLRETURN rc = SQLGetDescField_ptr(descriptor, recordNumber, fieldIdentifier, buffer, - static_cast(sizeof(buffer)), &length); - if (!SQL_SUCCEEDED(rc)) { - return rc; - } - if (length <= 0) { - value.clear(); - return rc; - } - size_t codeUnits = std::min(static_cast(length) / sizeof(SQLWCHAR), - (sizeof(buffer) / sizeof(buffer[0])) - 1); - value = dupeSqlWCharAsUtf16Le(buffer, codeUnits); - return rc; -} - -[[noreturn]] static void ThrowTableError(const SqlHandlePtr& statementHandle, SQLRETURN rc, +[[noreturn]] static void ThrowParameterError(const SqlHandlePtr& statementHandle, SQLRETURN rc, const std::string& operation) { ErrorInfo error = SQLCheckError_Wrap(SQL_HANDLE_STMT, statementHandle, rc); py::object raiseException = @@ -627,187 +463,6 @@ static SQLRETURN ReadDescriptorString(SQLHDESC descriptor, SQLSMALLINT recordNum throw std::runtime_error(operation); } -class TableMetadataScope { - public: - explicit TableMetadataScope(SQLHANDLE statementHandle) : handle(statementHandle) {} - ~TableMetadataScope() { - // Error paths can leave unread metadata; closing it can require network I/O. - if (!is_python_finalizing() && PyGILState_Check()) { - py::gil_scoped_release release; - SQLFreeStmt_ptr(handle, SQL_CLOSE); - } else { - SQLFreeStmt_ptr(handle, SQL_CLOSE); - } - SQLSetStmtAttr_ptr( - handle, SQL_SOPT_SS_NAME_SCOPE, - reinterpret_cast(static_cast(SQL_SS_NAME_SCOPE_TABLE)), - SQL_IS_INTEGER); - SQLSetStmtAttr_ptr(handle, SQL_ATTR_METADATA_ID, - reinterpret_cast(static_cast(SQL_FALSE)), - SQL_IS_INTEGER); - } - - private: - SQLHANDLE handle; -}; - -static std::vector LoadTableColumnMetadata(const SqlHandlePtr& metadataHandle, - const std::u16string& catalog, - const std::u16string& schema, - const std::u16string& typeName) { - if (!metadataHandle || !metadataHandle->get()) { - throw std::runtime_error("TVP column metadata requires a valid statement handle"); - } - - SQLHANDLE hMetadataStmt = metadataHandle->get(); - { - py::gil_scoped_release release; - SQLFreeStmt_ptr(hMetadataStmt, SQL_CLOSE); - SQLFreeStmt_ptr(hMetadataStmt, SQL_RESET_PARAMS); - } - - SQLRETURN rc = SQLSetStmtAttr_ptr(hMetadataStmt, SQL_ATTR_METADATA_ID, - reinterpret_cast(static_cast(SQL_TRUE)), - SQL_IS_INTEGER); - if (!SQL_SUCCEEDED(rc)) { - ThrowTableError(metadataHandle, rc, "Failed to enable metadata identifier mode"); - } - TableMetadataScope metadataScope(hMetadataStmt); - - rc = SQLSetStmtAttr_ptr( - hMetadataStmt, SQL_SOPT_SS_NAME_SCOPE, - reinterpret_cast(static_cast(SQL_SS_NAME_SCOPE_TABLE_TYPE)), - SQL_IS_INTEGER); - if (!SQL_SUCCEEDED(rc)) { - ThrowTableError(metadataHandle, rc, "Failed to select table-type metadata scope"); - } - { - py::gil_scoped_release release; - rc = SQLColumns_ptr(hMetadataStmt, - catalog.empty() ? nullptr : reinterpretU16stringAsSqlWChar(catalog), - catalog.empty() ? 0 : SQL_NTS, - schema.empty() ? nullptr : reinterpretU16stringAsSqlWChar(schema), - schema.empty() ? 0 : SQL_NTS, reinterpretU16stringAsSqlWChar(typeName), - SQL_NTS, nullptr, 0); - } - if (!SQL_SUCCEEDED(rc)) { - ThrowTableError(metadataHandle, rc, "Failed to discover TVP columns"); - } - - std::vector columns; - while (true) { - { - py::gil_scoped_release release; - rc = SQLFetch_ptr(hMetadataStmt); - } - if (rc == SQL_NO_DATA) { - break; - } - if (!SQL_SUCCEEDED(rc)) { - ThrowTableError(metadataHandle, rc, "Failed to fetch TVP column metadata"); - } - - SQLSMALLINT sqlType = SQL_UNKNOWN_TYPE; - SQLINTEGER columnSize = 0; - SQLSMALLINT decimalDigits = 0; - SQLLEN indicator = 0; - - { - py::gil_scoped_release release; - rc = SQLGetData_ptr(hMetadataStmt, 5, SQL_C_SSHORT, &sqlType, sizeof(sqlType), &indicator); - if (SQL_SUCCEEDED(rc)) { - rc = SQLGetData_ptr(hMetadataStmt, 7, SQL_C_SLONG, &columnSize, sizeof(columnSize), - &indicator); - } - if (SQL_SUCCEEDED(rc)) { - rc = SQLGetData_ptr(hMetadataStmt, 9, SQL_C_SSHORT, &decimalDigits, - sizeof(decimalDigits), &indicator); - if (indicator == SQL_NULL_DATA) { - decimalDigits = 0; - rc = SQL_SUCCESS; - } - } - } - if (!SQL_SUCCEEDED(rc)) { - ThrowTableError(metadataHandle, rc, "Failed to read TVP column metadata"); - } - - columns.push_back({ - sqlType, static_cast(std::max(columnSize, 0)), decimalDigits, - }); - } - - if (columns.empty()) { - throw std::runtime_error("No column metadata was returned for the table type"); - } - return columns; -} - -static SQLRETURN ResolveTableValuedParamTypes(SqlHandle& handle, SQLHANDLE hStmt, - const SqlHandlePtr& metadataHandle, - std::vector& paramInfos) { - for (size_t paramIndex = 0; paramIndex < paramInfos.size(); ++paramIndex) { - ParamInfo& info = paramInfos[paramIndex]; - if (!info.isTVP) { - continue; - } - - if (handle.tvpCache.find(static_cast(paramIndex)) == handle.tvpCache.end()) { - DescribedParamInfo described; - SQLSMALLINT nullable; - RETCODE rc; - { - py::gil_scoped_release release; - rc = SQLDescribeParam_ptr(hStmt, static_cast(paramIndex + 1), - &described.sqlType, &described.columnSize, - &described.decimalDigits, &nullable); - } - if (!SQL_SUCCEEDED(rc)) { - return rc; - } - if (described.sqlType != SQL_SS_TABLE) { - throw py::type_error( - "Nested row sequences can only be bound to a table-valued parameter"); - } - - SQLHDESC implementationDescriptor = nullptr; - rc = SQLGetStmtAttr_ptr(hStmt, SQL_ATTR_IMP_PARAM_DESC, &implementationDescriptor, 0, - nullptr); - if (!SQL_SUCCEEDED(rc)) { - return rc; - } - - std::u16string catalog; - std::u16string schema; - std::u16string typeName; - const SQLSMALLINT recordNumber = static_cast(paramIndex + 1); - rc = ReadDescriptorString(implementationDescriptor, recordNumber, - SQL_CA_SS_CATALOG_NAME, catalog); - if (SQL_SUCCEEDED(rc)) { - rc = ReadDescriptorString(implementationDescriptor, recordNumber, - SQL_CA_SS_SCHEMA_NAME, schema); - } - if (SQL_SUCCEEDED(rc)) { - rc = ReadDescriptorString(implementationDescriptor, recordNumber, - SQL_CA_SS_TYPE_NAME, typeName); - } - if (!SQL_SUCCEEDED(rc)) { - return rc; - } - if (typeName.empty()) { - throw std::runtime_error("The prepared statement did not expose a TVP type name"); - } - handle.tvpCache[static_cast(paramIndex)] = { - catalog, - schema, - typeName, - LoadTableColumnMetadata(metadataHandle, catalog, schema, typeName), - }; - } - } - return SQL_SUCCESS; -} - static SQLRETURN PreResolveUdtTypes(SQLHANDLE hStmt, std::vector& paramInfos) { bool hasUdt = false; for (const auto& info : paramInfos) { @@ -847,166 +502,10 @@ static SQLRETURN PreResolveUdtTypes(SQLHANDLE hStmt, std::vector& par return SQL_SUCCESS; } -static SQLRETURN BindTableValuedParameter(const SqlHandlePtr& statementHandle, SQLHANDLE hStmt, - int paramIndex, const py::handle& param, - std::vector>& paramBuffers, - const std::string& charEncoding) { - SqlHandle& handle = *statementHandle; - const py::sequence rows = py::reinterpret_borrow(param); - const SQLSMALLINT recordNumber = static_cast(paramIndex + 1); - - size_t columnCount = 0; - py::list columnValues; - std::vector columnInfos; - auto metadataEntry = handle.tvpCache.find(paramIndex); - if (metadataEntry == handle.tvpCache.end()) { - throw std::runtime_error("TVP column metadata was not resolved before binding"); - } - const TvpParamInfo& tableMetadata = metadataEntry->second; - const std::vector& declaredColumns = tableMetadata.columns; - if (!rows.empty()) { - const py::handle firstRow = rows[0]; - if (!PyList_Check(firstRow.ptr()) && !PyTuple_Check(firstRow.ptr())) { - throw py::type_error("TVP rows must be list or tuple objects"); - } - columnCount = py::len(firstRow); - if (columnCount == 0) { - throw py::value_error("TVP rows must contain at least one column"); - } - if (columnCount != declaredColumns.size()) { - throw py::value_error("TVP rows contain " + std::to_string(columnCount) + - " columns; the declared table type requires " + - std::to_string(declaredColumns.size())); - } - columnValues = py::list(columnCount); - for (size_t columnIndex = 0; columnIndex < columnCount; ++columnIndex) { - columnValues[columnIndex] = py::list(); - } - for (size_t rowIndex = 0; rowIndex < rows.size(); ++rowIndex) { - const py::sequence row = py::reinterpret_borrow(rows[rowIndex]); - if (!PyList_Check(row.ptr()) && !PyTuple_Check(row.ptr())) { - throw py::type_error("TVP rows must be list or tuple objects"); - } - if (static_cast(py::len(row)) != columnCount) { - throw py::value_error("TVP rows must all contain the same number of columns"); - } - for (size_t columnIndex = 0; columnIndex < columnCount; ++columnIndex) { - columnValues[columnIndex].cast().append(row[columnIndex]); - } - } - columnInfos = DetectTableColumnTypes(columnValues); - for (size_t columnIndex = 0; columnIndex < columnCount; ++columnIndex) { - ParamInfo& inferred = columnInfos[columnIndex]; - const DescribedParamInfo& declared = declaredColumns[columnIndex]; - // Bind the source representation; SQL Server converts it to the table's - // declared column type. Only an all-NULL column needs destination metadata. - if (inferred.paramCType == SQL_C_DEFAULT) { - inferred.paramSQLType = declared.sqlType; - inferred.columnSize = std::max(declared.columnSize, 1); - inferred.decimalDigits = declared.decimalDigits; - } else { - if (inferred.paramSQLType == SQL_WVARCHAR && inferred.columnSize > MAX_INLINE_CHAR) { - inferred.paramSQLType = SQL_WLONGVARCHAR; - } else if (inferred.paramSQLType == SQL_VARCHAR && - inferred.columnSize > MAX_INLINE_BINARY) { - inferred.paramSQLType = SQL_LONGVARCHAR; - } else if (inferred.paramSQLType == SQL_VARBINARY && - inferred.columnSize > MAX_INLINE_BINARY) { - inferred.paramSQLType = SQL_LONGVARBINARY; - } - } - } - } - - auto* typeName = AllocateParamBuffer(paramBuffers, tableMetadata.typeName); - auto* schema = AllocateParamBuffer(paramBuffers, tableMetadata.schema); - auto* rowCount = AllocateParamBuffer(paramBuffers); - // TVPs overload ColumnSize with row capacity and the indicator with the rows available. - *rowCount = rows.empty() ? SQL_DEFAULT_PARAM : static_cast(rows.size()); - - SQLRETURN rc = SQLBindParameter_ptr( - hStmt, static_cast(recordNumber), SQL_PARAM_INPUT, SQL_C_BINARY, SQL_SS_TABLE, - static_cast(rows.size()), 0, typeName->data(), - static_cast(typeName->size() * sizeof(SQLWCHAR)), rowCount); - if (!SQL_SUCCEEDED(rc)) { - return rc; - } - if (!schema->empty()) { - SQLHDESC implementationDescriptor = nullptr; - rc = SQLGetStmtAttr_ptr(hStmt, SQL_ATTR_IMP_PARAM_DESC, &implementationDescriptor, 0, - nullptr); - if (!SQL_SUCCEEDED(rc)) { - return rc; - } - rc = SQLSetDescField_ptr(implementationDescriptor, recordNumber, SQL_CA_SS_SCHEMA_NAME, - reinterpretU16stringAsSqlWChar(*schema), - static_cast(schema->size() * sizeof(SQLWCHAR))); - if (!SQL_SUCCEEDED(rc)) { - return rc; - } - } - if (rows.empty()) { - return rc; - } - - rc = SQLSetStmtAttr_ptr(hStmt, SQL_SOPT_SS_PARAM_FOCUS, - reinterpret_cast(static_cast(recordNumber)), - SQL_IS_INTEGER); - if (!SQL_SUCCEEDED(rc)) { - return rc; - } - - try { - rc = BindParameterArray(handle, hStmt, columnValues, columnInfos, rows.size(), - paramBuffers, charEncoding); - if (!SQL_SUCCEEDED(rc)) { - // Successful focus restoration clears the original statement diagnostic. - ThrowTableError(statementHandle, rc, "Failed to bind TVP columns"); - } - } catch (...) { - SQLSetStmtAttr_ptr(hStmt, SQL_SOPT_SS_PARAM_FOCUS, nullptr, SQL_IS_INTEGER); - throw; - } - return SQLSetStmtAttr_ptr(hStmt, SQL_SOPT_SS_PARAM_FOCUS, nullptr, SQL_IS_INTEGER); -} - -// Given a list of parameters and their ParamInfo, calls SQLBindParameter on -// each of them with appropriate arguments -SQLRETURN BindParameters(const SqlHandlePtr& statementHandle, SQLHANDLE hStmt, const py::list& params, - std::vector& paramInfos, - std::vector>& paramBuffers, - const std::string& charEncoding = "utf-8") { - PERF_TIMER("BindParameters"); - SqlHandle& handle = *statementHandle; - LOG("BindParameters: Starting parameter binding for statement handle %p " - "with %zu parameters", - (void*)hStmt, params.size()); - - SQLRETURN describeRc = PreResolveUdtTypes(hStmt, paramInfos); - if (!SQL_SUCCEEDED(describeRc)) return describeRc; - // GH-627: resolve unknown NULL param SQL types before binding any param. - PreResolveUnknownNullTypes(handle, hStmt, paramInfos, ¶ms); - for (int paramIndex = 0; paramIndex < params.size(); paramIndex++) { - const auto& param = params[paramIndex]; - ParamInfo& paramInfo = paramInfos[paramIndex]; - LOG("BindParameters: Processing param[%d] - C_Type=%d, SQL_Type=%d, " - "ColumnSize=%lu, DecimalDigits=%d, InputOutputType=%d", - paramIndex, paramInfo.paramCType, paramInfo.paramSQLType, - (unsigned long)paramInfo.columnSize, paramInfo.decimalDigits, - paramInfo.inputOutputType); - void* dataPtr = nullptr; - SQLLEN bufferLength = 0; - SQLLEN* strLenOrIndPtr = nullptr; - - if (paramInfo.isTVP) { - RETCODE rc = BindTableValuedParameter(statementHandle, hStmt, paramIndex, param, - paramBuffers, charEncoding); - if (!SQL_SUCCEEDED(rc)) { - return rc; - } - continue; - } - +static void PrepareParamBuffer(const py::handle& param, ParamInfo& paramInfo, + std::vector>& paramBuffers, + const std::string& charEncoding, int paramIndex, + void*& dataPtr, SQLLEN& bufferLength, SQLLEN*& strLenOrIndPtr) { // TODO: Add more data types like money, guid, interval, TVPs etc. switch (paramInfo.paramCType) { case SQL_C_CHAR: { @@ -1019,7 +518,7 @@ SQLRETURN BindParameters(const SqlHandlePtr& statementHandle, SQLHANDLE hStmt, c "(Data-At-Execution) for large string streaming", paramIndex); dataPtr = - const_cast(reinterpret_cast(¶mInfos[paramIndex])); + const_cast(reinterpret_cast(¶mInfo)); strLenOrIndPtr = AllocateParamBuffer(paramBuffers); *strLenOrIndPtr = SQL_LEN_DATA_AT_EXEC(0); bufferLength = 0; @@ -1080,7 +579,7 @@ SQLRETURN BindParameters(const SqlHandlePtr& statementHandle, SQLHANDLE hStmt, c "for VARBINARY(MAX) streaming", paramIndex); dataPtr = - const_cast(reinterpret_cast(¶mInfos[paramIndex])); + const_cast(reinterpret_cast(¶mInfo)); strLenOrIndPtr = AllocateParamBuffer(paramBuffers); *strLenOrIndPtr = SQL_LEN_DATA_AT_EXEC(0); bufferLength = 0; @@ -1115,7 +614,7 @@ SQLRETURN BindParameters(const SqlHandlePtr& statementHandle, SQLHANDLE hStmt, c "NVARCHAR(MAX) streaming", paramIndex); dataPtr = - const_cast(reinterpret_cast(¶mInfos[paramIndex])); + const_cast(reinterpret_cast(¶mInfo)); strLenOrIndPtr = AllocateParamBuffer(paramBuffers); *strLenOrIndPtr = SQL_LEN_DATA_AT_EXEC(0); bufferLength = 0; @@ -1233,9 +732,9 @@ SQLRETURN BindParameters(const SqlHandlePtr& statementHandle, SQLHANDLE hStmt, c ThrowStdException(MakeParamMismatchErrorStr(paramInfo.paramCType, paramIndex)); } int year = param.attr("year").cast(); - if (year < 1753 || year > 9999) { + if (year < 1 || year > 9999) { ThrowStdException("Date out of range for SQL Server " - "(1753-9999) at paramIndex " + + "(1-9999) at paramIndex " + std::to_string(paramIndex)); } // TODO: can be moved to python by registering SQL_DATE_STRUCT @@ -1393,6 +892,48 @@ SQLRETURN BindParameters(const SqlHandlePtr& statementHandle, SQLHANDLE hStmt, c ThrowStdException(errorString.str()); } } +} + +#include "tvp.hpp" + +// Given a list of parameters and their ParamInfo, calls SQLBindParameter on +// each of them with appropriate arguments +SQLRETURN BindParameters(const SqlHandlePtr& statementHandle, SQLHANDLE hStmt, const py::list& params, + std::vector& paramInfos, + std::vector>& paramBuffers, + const std::string& charEncoding = "utf-8") { + PERF_TIMER("BindParameters"); + SqlHandle& handle = *statementHandle; + LOG("BindParameters: Starting parameter binding for statement handle %p " + "with %zu parameters", + (void*)hStmt, params.size()); + + SQLRETURN describeRc = PreResolveUdtTypes(hStmt, paramInfos); + if (!SQL_SUCCEEDED(describeRc)) return describeRc; + // GH-627: resolve unknown NULL param SQL types before binding any param. + PreResolveUnknownNullTypes(handle, hStmt, paramInfos, ¶ms); + for (int paramIndex = 0; paramIndex < params.size(); paramIndex++) { + const auto& param = params[paramIndex]; + ParamInfo& paramInfo = paramInfos[paramIndex]; + LOG("BindParameters: Processing param[%d] - C_Type=%d, SQL_Type=%d, " + "ColumnSize=%lu, DecimalDigits=%d, InputOutputType=%d", + paramIndex, paramInfo.paramCType, paramInfo.paramSQLType, + (unsigned long)paramInfo.columnSize, paramInfo.decimalDigits, + paramInfo.inputOutputType); + void* dataPtr = nullptr; + SQLLEN bufferLength = 0; + SQLLEN* strLenOrIndPtr = nullptr; + + if (paramInfo.isTVP) { + RETCODE rc = BindTableValuedParameter(statementHandle, paramIndex, param, paramInfo); + if (!SQL_SUCCEEDED(rc)) { + return rc; + } + continue; + } + + PrepareParamBuffer(param, paramInfo, paramBuffers, charEncoding, paramIndex, + dataPtr, bufferLength, strLenOrIndPtr); assert(SQLBindParameter_ptr && SQLGetStmtAttr_ptr && SQLSetDescField_ptr); RETCODE rc; { @@ -1979,7 +1520,6 @@ DriverHandle LoadDriverOrThrowException() { SQLRowCount_ptr = GetFunctionPointer(handle, "SQLRowCount"); SQLGetStmtAttr_ptr = GetFunctionPointer(handle, "SQLGetStmtAttrW"); SQLSetDescField_ptr = GetFunctionPointer(handle, "SQLSetDescFieldW"); - SQLGetDescField_ptr = GetFunctionPointer(handle, "SQLGetDescFieldW"); SQLFetch_ptr = GetFunctionPointer(handle, "SQLFetch"); SQLFetchScroll_ptr = GetFunctionPointer(handle, "SQLFetchScroll"); @@ -2019,7 +1559,7 @@ DriverHandle LoadDriverOrThrowException() { SQLAllocHandle_ptr && SQLSetEnvAttr_ptr && SQLSetConnectAttr_ptr && SQLSetStmtAttr_ptr && SQLGetConnectAttr_ptr && SQLDriverConnect_ptr && SQLExecDirect_ptr && SQLPrepare_ptr && SQLBindParameter_ptr && SQLExecute_ptr && SQLRowCount_ptr && SQLGetStmtAttr_ptr && - SQLSetDescField_ptr && SQLGetDescField_ptr && SQLFetch_ptr && SQLFetchScroll_ptr && + SQLSetDescField_ptr && SQLFetch_ptr && SQLFetchScroll_ptr && SQLGetData_ptr && SQLNumResultCols_ptr && SQLBindCol_ptr && SQLDescribeCol_ptr && SQLMoreResults_ptr && SQLColAttribute_ptr && SQLEndTran_ptr && SQLDisconnect_ptr && SQLFreeHandle_ptr && SQLFreeStmt_ptr && SQLGetDiagRec_ptr && SQLGetInfo_ptr && @@ -2096,7 +1636,7 @@ std::unique_lock SqlHandle::lockForCleanup() const { } SQLHANDLE SqlHandle::get() const { - return _handle; + return _failedParameterBuffers.empty() ? _handle : nullptr; } SQLSMALLINT SqlHandle::type() const { @@ -2604,7 +2144,7 @@ SQLRETURN SQLTables_wrap(SqlHandlePtr StatementHandle, const std::u16string& cat // When false and not prepared, throws (matching slow path behavior). // --------------------------------------------------------------------------- SQLRETURN SQLExecute_wrap(const SqlHandlePtr statementHandle, - const SqlHandlePtr metadataStatementHandle, const std::u16string& query, + const std::u16string& query, py::list params, const py::object& input_sizes, py::list is_stmt_prepared, bool use_prepare, const py::dict& encoding_settings) { PERF_TIMER("SQLExecute_wrap"); @@ -2667,13 +2207,10 @@ SQLRETURN SQLExecute_wrap(const SqlHandlePtr statementHandle, } } - rc = ResolveTableValuedParamTypes(*statementHandle, hStmt, metadataStatementHandle, paramInfos); - if (!SQL_SUCCEEDED(rc)) - return rc; - std::vector> paramBuffers; + try { rc = BindParameters(statementHandle, hStmt, params, paramInfos, paramBuffers, charEncoding); - if (!SQL_SUCCEEDED(rc)) return rc; + if (!SQL_SUCCEEDED(rc)) ThrowParameterError(statementHandle, rc, "Binding parameters"); { py::gil_scoped_release release; @@ -2687,8 +2224,17 @@ SQLRETURN SQLExecute_wrap(const SqlHandlePtr statementHandle, if (rc == SQL_NEED_DATA) { SQLPOINTER paramToken = nullptr; auto putData = [&](SQLPOINTER data, SQLLEN len) { - py::gil_scoped_release release; - return SQLPutData_ptr(hStmt, data, len); + if (statementHandle->get() != hStmt || statementHandle->isImplicitlyFreed()) + throw std::runtime_error("Statement closed while producing parameter data"); + SQLRETURN result; + { + py::gil_scoped_release release; + result = SQLPutData_ptr(hStmt, data, len); + } + if (!SQL_SUCCEEDED(result)) + ThrowParameterError(statementHandle, result, + "Sending parameter data (length " + std::to_string(len) + ")"); + return result; }; while (true) { { @@ -2697,6 +2243,11 @@ SQLRETURN SQLExecute_wrap(const SqlHandlePtr statementHandle, } if (rc != SQL_NEED_DATA) break; + if (SendTableData(paramToken, paramInfos, putData, rc)) { + if (!SQL_SUCCEEDED(rc)) ThrowParameterError(statementHandle, rc, "Sending TVP data"); + continue; + } + // The DAE token is the ¶mInfos[i] we handed to SQLBindParameter as the // parameter value (see BindParameters), and paramInfos is sized up front and // never reallocated, so the token casts straight back to its ParamInfo instead @@ -2710,8 +2261,7 @@ SQLRETURN SQLExecute_wrap(const SqlHandlePtr statementHandle, } PyObject* pyObj = matchedInfo->dataPtr.ptr(); if (!pyObj || pyObj == Py_None) { - py::gil_scoped_release release; - SQLPutData_ptr(hStmt, nullptr, 0); + putData(nullptr, 0); continue; } @@ -2761,22 +2311,43 @@ SQLRETURN SQLExecute_wrap(const SqlHandlePtr statementHandle, ThrowStdException("SQLExecute: DAE only supported for str or bytes"); } } - if (!SQL_SUCCEEDED(rc) && rc != SQL_NO_DATA) return rc; + if (!SQL_SUCCEEDED(rc) && rc != SQL_NO_DATA) + ThrowParameterError(statementHandle, rc, "Executing parameters"); } - if (!SQL_SUCCEEDED(rc) && rc != SQL_NO_DATA) return rc; + if (!SQL_SUCCEEDED(rc) && rc != SQL_NO_DATA) + ThrowParameterError(statementHandle, rc, "Executing parameters"); // Unbind parameter buffers before they go out of scope. - // Not called on error paths — diagnostics must remain readable. SQLRETURN exec_rc = rc; - SQLFreeStmt_ptr(hStmt, SQL_RESET_PARAMS); + rc = SQLFreeStmt_ptr(hStmt, SQL_RESET_PARAMS); + if (!SQL_SUCCEEDED(rc)) ThrowParameterError(statementHandle, rc, "Resetting parameters"); return exec_rc; + } catch (...) { + // Convert diagnostics to the original Python exception before cleanup + // can overwrite them. Keep every buffer alive until NEED_DATA is left. + if (statementHandle->get() == hStmt && !statementHandle->isImplicitlyFreed()) { + SQLRETURN resetResult; + { + py::gil_scoped_release release; + SQLCancel_ptr(hStmt); + resetResult = SQLFreeStmt_ptr(hStmt, SQL_RESET_PARAMS); + } + SQLRETURN freeResult = statementHandle->freeHandle(); + if (!SQL_SUCCEEDED(resetResult) && !SQL_SUCCEEDED(freeResult)) { + paramBuffers.push_back( + std::make_shared>(std::move(paramInfos))); + statementHandle->retainFailedParameters(std::move(paramBuffers)); + } + } + throw; + } } SQLRETURN BindParameterArray(SqlHandle& handle, SQLHANDLE hStmt, const py::list& columnwise_params, std::vector& paramInfos, size_t paramSetSize, std::vector>& paramBuffers, - const std::string& charEncoding) { + const std::string& charEncoding = "utf-8") { PERF_TIMER("BindParameterArray"); LOG("BindParameterArray: Starting column-wise array binding - " "param_count=%zu, param_set_size=%zu", @@ -2869,8 +2440,7 @@ SQLRETURN BindParameterArray(SqlHandle& handle, SQLHANDLE hStmt, const py::list& } std::memcpy(wcharArray + i * (info.columnSize + 1), wstr.c_str(), (wstr.length() + 1) * sizeof(SQLWCHAR)); - strLenOrIndArray[i] = - static_cast(wstr.length() * sizeof(SQLWCHAR)); + strLenOrIndArray[i] = SQL_NTS; } } LOG("BindParameterArray: SQL_C_WCHAR bound - " @@ -2945,19 +2515,17 @@ SQLRETURN BindParameterArray(SqlHandle& handle, SQLHANDLE hStmt, const py::list& } case SQL_C_CHAR: case SQL_C_BINARY: { - const SQLULEN valueBufferSize = - info.bufferSize > 0 ? info.bufferSize : info.columnSize; LOG("BindParameterArray: Binding SQL_C_CHAR/BINARY array - " "param_index=%d, count=%zu, column_size=%zu, encoding='%s'", - paramIndex, paramSetSize, valueBufferSize, charEncoding.c_str()); + paramIndex, paramSetSize, info.columnSize, charEncoding.c_str()); char* charArray = AllocateParamBufferArray( - tempBuffers, paramSetSize * (valueBufferSize + 1)); + tempBuffers, paramSetSize * (info.columnSize + 1)); strLenOrIndArray = AllocateParamBufferArray(tempBuffers, paramSetSize); for (size_t i = 0; i < paramSetSize; ++i) { if (columnValues[i].is_none()) { strLenOrIndArray[i] = SQL_NULL_DATA; - std::memset(charArray + i * (valueBufferSize + 1), 0, - valueBufferSize + 1); + std::memset(charArray + i * (info.columnSize + 1), 0, + info.columnSize + 1); } else { std::string encodedStr; @@ -2987,15 +2555,15 @@ SQLRETURN BindParameterArray(SqlHandle& handle, SQLHANDLE hStmt, const py::list& encodedStr = columnValues[i].cast(); } - if (encodedStr.size() > valueBufferSize) { + if (encodedStr.size() > info.columnSize) { LOG("BindParameterArray: String/binary too " "long - param_index=%d, row=%zu, size=%zu, " "max=%zu", - paramIndex, i, encodedStr.size(), valueBufferSize); + paramIndex, i, encodedStr.size(), info.columnSize); ThrowStdException("Input exceeds column size at index " + std::to_string(i)); } - std::memcpy(charArray + i * (valueBufferSize + 1), encodedStr.c_str(), + std::memcpy(charArray + i * (info.columnSize + 1), encodedStr.c_str(), encodedStr.size()); strLenOrIndArray[i] = static_cast(encodedStr.size()); } @@ -3004,7 +2572,7 @@ SQLRETURN BindParameterArray(SqlHandle& handle, SQLHANDLE hStmt, const py::list& "param_index=%d", paramIndex); dataPtr = charArray; - bufferLength = valueBufferSize + 1; + bufferLength = info.columnSize + 1; break; } case SQL_C_BIT: { @@ -6989,7 +6557,7 @@ PYBIND11_MODULE(ddbc_bindings, m) { m.def("DDBCSQLExecDirect", &SQLExecDirect_wrap, "Execute a SQL query directly"); m.def("DDBCSQLExecute", &SQLExecute_wrap, "DetectParamTypes + BindParameters + SQLExecute all in C++", py::arg("statementHandle"), - py::arg("metadataStatementHandle"), py::arg("query"), py::arg("params"), + py::arg("query"), py::arg("params"), py::arg("inputSizes"), py::arg("isStmtPrepared"), py::arg("usePrepare"), py::arg("encodingSettings")); m.def("SQLExecuteMany", &SQLExecuteMany_wrap, "Execute statement with multiple parameter sets", diff --git a/mssql_python/pybind/ddbc_bindings.h b/mssql_python/pybind/ddbc_bindings.h index 6f0c9ceef..e0d47ad40 100644 --- a/mssql_python/pybind/ddbc_bindings.h +++ b/mssql_python/pybind/ddbc_bindings.h @@ -57,10 +57,6 @@ using py::literals::operator""_a; #define SQL_SS_TABLE (-153) #define SQL_CA_SS_VARIANT_TYPE (1215) #define SQL_SOPT_SS_PARAM_FOCUS (1236) -#define SQL_SOPT_SS_NAME_SCOPE (1237) -#define SQL_SS_NAME_SCOPE_TABLE (0L) -#define SQL_SS_NAME_SCOPE_TABLE_TYPE (1L) -#define SQL_CA_SS_CATALOG_NAME (1225) #define SQL_CA_SS_SCHEMA_NAME (1226) #define SQL_CA_SS_TYPE_NAME (1227) @@ -97,8 +93,6 @@ typedef SQLRETURN(SQL_API* SQLExecuteFunc)(SQLHANDLE); typedef SQLRETURN(SQL_API* SQLRowCountFunc)(SQLHSTMT, SQLLEN*); typedef SQLRETURN(SQL_API* SQLSetDescFieldFunc)(SQLHDESC, SQLSMALLINT, SQLSMALLINT, SQLPOINTER, SQLINTEGER); -typedef SQLRETURN(SQL_API* SQLGetDescFieldFunc)(SQLHDESC, SQLSMALLINT, SQLSMALLINT, SQLPOINTER, - SQLINTEGER, SQLINTEGER*); typedef SQLRETURN(SQL_API* SQLGetStmtAttrFunc)(SQLHSTMT, SQLINTEGER, SQLPOINTER, SQLINTEGER, SQLINTEGER*); @@ -186,7 +180,6 @@ extern SQLBindParameterFunc SQLBindParameter_ptr; extern SQLExecuteFunc SQLExecute_ptr; extern SQLRowCountFunc SQLRowCount_ptr; extern SQLSetDescFieldFunc SQLSetDescField_ptr; -extern SQLGetDescFieldFunc SQLGetDescField_ptr; extern SQLGetStmtAttrFunc SQLGetStmtAttr_ptr; // Data retrieval APIs @@ -299,13 +292,6 @@ struct DescribedParamInfo { SQLSMALLINT decimalDigits; }; -struct TvpParamInfo { - std::u16string catalog; - std::u16string schema; - std::u16string typeName; - std::vector columns; -}; - struct ConnectionCleanupState { std::mutex mutex; bool disconnected = false; // Protected by mutex, shared with every child. @@ -329,6 +315,9 @@ class SqlHandle { // unavailable. void cancel(); bool isImplicitlyFreed() const { return _implicitly_freed; } + void retainFailedParameters(std::vector> buffers) { + _failedParameterBuffers = std::move(buffers); + } // Mark this handle as implicitly freed (freed by parent handle) // This prevents double-free attempts when the ODBC driver automatically @@ -350,11 +339,7 @@ class SqlHandle { // on handle free. No mutex needed — ODBC statement handles are not // thread-safe by spec (same assumption as the rest of the driver). std::unordered_map describeCache; - std::unordered_map tvpCache; - void clearDescribeCache() { - describeCache.clear(); - tvpCache.clear(); - } + void clearDescribeCache() { describeCache.clear(); } ResultMetadataCache resultMetadata; private: @@ -364,6 +349,9 @@ class SqlHandle { SQLHANDLE _handle; bool _implicitly_freed = false; // Tracks if handle was freed by parent std::shared_ptr _cleanupState; + // Only populated if both parameter reset and handle release fail. The + // handle is then unusable, but its bound storage lives through teardown. + std::vector> _failedParameterBuffers; }; using SqlHandlePtr = std::shared_ptr; diff --git a/mssql_python/pybind/param_detect.hpp b/mssql_python/pybind/param_detect.hpp index feb0aaadc..61f2c0a68 100644 --- a/mssql_python/pybind/param_detect.hpp +++ b/mssql_python/pybind/param_detect.hpp @@ -46,16 +46,17 @@ #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wattributes" #endif +namespace { struct TableParamState; } struct ParamInfo { SQLSMALLINT inputOutputType = SQL_PARAM_INPUT; SQLSMALLINT paramCType = SQL_C_DEFAULT; SQLSMALLINT paramSQLType = SQL_UNKNOWN_TYPE; SQLULEN columnSize = 0; - SQLULEN bufferSize = 0; SQLSMALLINT decimalDigits = 0; SQLLEN strLenOrInd = 0; // Required for DAE bool isDAE = false; // Indicates if we need to stream bool isTVP = false; + std::shared_ptr table; // Strong reference to the Python object for DAE (data-at-execution) streaming. // py::object owns the refcount, so the compiler-generated destructor, copy and // move operations are all correct and this struct needs no rule-of-five. @@ -314,25 +315,13 @@ inline std::vector DetectParamTypes(PyObject* params, PyObject* input info.isDAE = false; info.isTVP = false; - PyObject* obj = PyList_GET_ITEM(params, i); - if (PyList_Check(obj) || PyTuple_Check(obj)) { - if (i < inputSizeCount) { - throw py::type_error("setinputsizes cannot override table-valued parameters"); - } - info.paramSQLType = SQL_SS_TABLE; - info.paramCType = SQL_C_BINARY; - info.columnSize = static_cast(PySequence_Size(obj)); - info.decimalDigits = 0; - info.isTVP = true; - info.dataPtr = borrow(obj); - continue; - } - if (i < inputSizeCount) { ApplyInputSizeOverride(params, PyList_GET_ITEM(inputSizes, i), i, info); continue; } + PyObject* obj = PyList_GET_ITEM(params, i); + // --- None --- if (obj == Py_None) { info.paramSQLType = SQL_UNKNOWN_TYPE; @@ -650,6 +639,15 @@ inline std::vector DetectParamTypes(PyObject* params, PyObject* input continue; } + py::object tvpType = py::module_::import("mssql_python.table_valued_parameter") + .attr("TableValuedParameter"); + if (py::isinstance(borrow(obj), tvpType)) { + info.paramSQLType = SQL_SS_TABLE; + info.paramCType = SQL_C_BINARY; + info.isTVP = true; + continue; + } + // --- Unknown type: raise TypeError (matches Python _map_sql_type) --- throw py::type_error( "Unsupported parameter type: The driver cannot safely convert it to a SQL type."); diff --git a/mssql_python/pybind/tvp.hpp b/mssql_python/pybind/tvp.hpp new file mode 100644 index 000000000..f7a762cd5 --- /dev/null +++ b/mssql_python/pybind/tvp.hpp @@ -0,0 +1,172 @@ +// One-row ODBC variable-row binding. Included after PrepareParamBuffer so +// scalar and TVP cells share the same conversion and buffer ownership rules. +#pragma once + +struct TableParamState { + py::object value; + py::object iterator; + py::list sizes; + py::list row; + std::u16string typeName; + std::u16string schema; + std::vector columns; + std::vector indicators; + std::vector data; + std::vector lengths; + std::vector> rowBuffers; + SQLLEN indicator = SQL_DATA_AT_EXEC; + size_t rowIndex = 0; + bool finished = false; + + explicit TableParamState(const py::handle& table) + : value(py::reinterpret_borrow(table)), + sizes(table.attr("_odbc_columns")().cast()), + typeName(table.attr("type_name").cast()), + schema(table.attr("schema").cast()), + columns(sizes.size()), indicators(sizes.size(), SQL_DATA_AT_EXEC), + data(sizes.size()), lengths(sizes.size()) { + iterator = steal(PyObject_GetIter(table.attr("rows").ptr())); + if (!iterator) throw py::error_already_set(); + for (size_t i = 0; i < sizes.size(); ++i) { + py::tuple spec = sizes[i].cast(); + columns[i].paramSQLType = spec[0].cast(); + columns[i].paramCType = spec[1].cast(); + columns[i].columnSize = spec[2].cast(); + columns[i].decimalDigits = spec[3].cast(); + } + } + + bool next() { + // Drop the previous window before entering user code for the next row. + row = py::list(); + rowBuffers.clear(); + if (PyErr_CheckSignals() != 0) throw py::error_already_set(); + py::object item = steal(PyIter_Next(iterator.ptr())); + if (!item) { + if (PyErr_Occurred()) throw py::error_already_set(); + finished = true; + return false; + } + py::object checked = value.attr("_validate_row")(item, rowIndex++); + row = py::reinterpret_steal(PySequence_List(checked.ptr())); + if (!row) throw py::error_already_set(); + for (size_t i = 0; i < columns.size(); ++i) { + const ParamInfo& declared = columns[i]; + if (row[i].is_none()) { + indicators[i] = SQL_NULL_DATA; + continue; + } + if (declared.paramCType == SQL_C_GUID) { + row[i] = row[i].attr("bytes_le"); + } + // ApplyInputSizeOverride preserves decimal coefficients (no float or + // decimal-context arithmetic) and fractional time via isoformat. + ParamInfo converted; + ApplyInputSizeOverride(row.ptr(), sizes[i].ptr(), i, converted); + converted.isDAE = false; + data[i] = nullptr; + lengths[i] = 0; + SQLLEN* lengthOrIndicator = nullptr; + PrepareParamBuffer(row[i], converted, rowBuffers, "utf-8", static_cast(i), + data[i], lengths[i], lengthOrIndicator); + if (lengthOrIndicator) lengths[i] = *lengthOrIndicator; + switch (declared.paramCType) { + case SQL_C_BIT: case SQL_C_TINYINT: case SQL_C_UTINYINT: + lengths[i] = sizeof(SQLCHAR); break; + case SQL_C_SHORT: lengths[i] = sizeof(SQLSMALLINT); break; + case SQL_C_LONG: lengths[i] = sizeof(SQLINTEGER); break; + case SQL_C_SBIGINT: lengths[i] = sizeof(int64_t); break; + case SQL_C_FLOAT: lengths[i] = sizeof(float); break; + case SQL_C_DOUBLE: lengths[i] = sizeof(double); break; + case SQL_C_TYPE_DATE: lengths[i] = sizeof(SQL_DATE_STRUCT); break; + case SQL_C_TYPE_TIME: lengths[i] = sizeof(SQL_TIME_STRUCT); break; + case SQL_C_TYPE_TIMESTAMP: lengths[i] = sizeof(SQL_TIMESTAMP_STRUCT); break; + } + indicators[i] = SQL_DATA_AT_EXEC; + } + return true; + } +}; + +static SQLRETURN BindTableValuedParameter(const SqlHandlePtr& handle, int paramIndex, + const py::handle& value, ParamInfo& info) { + info.table = std::make_shared(value); + TableParamState& table = *info.table; + SQLHANDLE hStmt = handle->get(); + if (!hStmt || handle->isImplicitlyFreed()) + throw std::runtime_error("Statement closed while initializing TVP rows"); + SQLSMALLINT record = static_cast(paramIndex + 1); + // msodbcsql18 enforces ColumnSize as a cumulative row ceiling even for + // variable-row DAE (1 rejects the second row). This is a descriptor limit, + // not an allocation: all columns are DAE tokens with one row of storage. + SQLRETURN rc = SQLBindParameter_ptr( + hStmt, record, SQL_PARAM_INPUT, SQL_C_BINARY, SQL_SS_TABLE, + std::numeric_limits::max(), 0, + nullptr, 0, &table.indicator); + if (!SQL_SUCCEEDED(rc)) return rc; + SQLHDESC descriptor = nullptr; + rc = SQLGetStmtAttr_ptr(hStmt, SQL_ATTR_IMP_PARAM_DESC, &descriptor, 0, nullptr); + if (!SQL_SUCCEEDED(rc)) return rc; + rc = SQLSetDescField_ptr(descriptor, record, SQL_CA_SS_TYPE_NAME, + table.typeName.data(), + static_cast(table.typeName.size() * sizeof(SQLWCHAR))); + if (!SQL_SUCCEEDED(rc)) return rc; + rc = SQLSetDescField_ptr(descriptor, record, SQL_CA_SS_SCHEMA_NAME, + table.schema.data(), + static_cast(table.schema.size() * sizeof(SQLWCHAR))); + if (!SQL_SUCCEEDED(rc)) return rc; + rc = SQLGetStmtAttr_ptr(hStmt, SQL_ATTR_APP_PARAM_DESC, &descriptor, 0, nullptr); + if (!SQL_SUCCEEDED(rc)) return rc; + rc = SQLSetDescField_ptr(descriptor, record, SQL_DESC_DATA_PTR, &table, 0); + if (!SQL_SUCCEEDED(rc)) return rc; + rc = SQLSetStmtAttr_ptr(hStmt, SQL_SOPT_SS_PARAM_FOCUS, + reinterpret_cast(static_cast(record)), + SQL_IS_INTEGER); + if (!SQL_SUCCEEDED(rc)) return rc; + try { + for (size_t i = 0; i < table.columns.size(); ++i) { + const ParamInfo& column = table.columns[i]; + rc = SQLBindParameter_ptr( + hStmt, static_cast(i + 1), SQL_PARAM_INPUT, + column.paramCType, column.paramSQLType, column.columnSize, column.decimalDigits, + &table.columns[i], 0, &table.indicators[i]); + if (!SQL_SUCCEEDED(rc)) ThrowParameterError(handle, rc, "Failed to bind TVP column"); + } + } catch (...) { + SQLSetStmtAttr_ptr(hStmt, SQL_SOPT_SS_PARAM_FOCUS, nullptr, SQL_IS_INTEGER); + throw; + } + return SQLSetStmtAttr_ptr(hStmt, SQL_SOPT_SS_PARAM_FOCUS, nullptr, SQL_IS_INTEGER); +} + +template +static bool SendTableData(SQLPOINTER token, std::vector& params, + PutData putData, SQLRETURN& rc) { + for (auto& param : params) { + if (!param.table) continue; + auto& table = *param.table; + if (token == &table) { + if (table.finished) throw std::runtime_error("TVP requested rows after completion"); + SQLLEN count = table.next() ? 1 : 0; + // For the table token, the driver interprets DataPtr as the + // row count (not a pointer to row data). Column tokens carry data. + rc = putData(reinterpret_cast(static_cast(count)), count); + return true; + } + for (size_t i = 0; i < table.columns.size(); ++i) { + if (token != &table.columns[i]) continue; + if (table.indicators[i] == SQL_NULL_DATA) { + rc = putData(nullptr, SQL_NULL_DATA); + return true; + } + const auto cType = table.columns[i].paramCType; + if (cType == SQL_C_WCHAR || cType == SQL_C_CHAR || cType == SQL_C_BINARY) { + rc = stream_dae_chunks(table.data[i], static_cast(table.lengths[i]), putData); + } else { + rc = putData(table.data[i], table.lengths[i]); + } + return true; + } + } + return false; +} diff --git a/mssql_python/table_valued_parameter.py b/mssql_python/table_valued_parameter.py new file mode 100644 index 000000000..770324521 --- /dev/null +++ b/mssql_python/table_valued_parameter.py @@ -0,0 +1,172 @@ +"""Explicit, transport-neutral input values for SQL Server table types.""" + +from collections.abc import AsyncIterable, Iterator +from datetime import date, datetime, time +from decimal import Decimal +from uuid import UUID + +from .constants import ConstantsDDBC as SQL + + +class TableValuedParameter: + """A table type, its ordered column declarations, and an unconsumed row source. + + Columns are ``(name, sql_type[, size_or_precision[, scale]])`` tuples. + Variable-length columns require a size (0 means MAX); DECIMAL/NUMERIC + require precision and scale. Temporal scale defaults to 7. + ``type_name`` accepts an unquoted ``schema.Type`` or a type name with the + separate ``schema`` keyword (default ``dbo``). Binding is positional. + """ + + __slots__ = ("_type_name", "_schema", "_columns", "_rows") + + def __init__(self, type_name, columns, rows, *, schema=None): + if not isinstance(type_name, str): + raise TypeError("TVP type_name must be a string") + if "." in type_name: + if schema is not None or type_name.count(".") != 1: + raise ValueError("Specify TVP schema once: schema.Type or schema=...") + schema, type_name = type_name.split(".") + schema = "dbo" if schema is None else schema + for name in (type_name, schema): + if not isinstance(name, str): + raise TypeError("TVP schema and type name must be strings") + if ( + not name.strip() + or any(char in name for char in '\x00.[]"') + or len(name.encode("utf-16le")) > 256 + ): + raise ValueError("TVP names must be unquoted identifiers of 1–128 UTF-16 units") + if not isinstance(columns, (list, tuple)) or not columns: + raise TypeError("TVP columns must be a non-empty list or tuple") + normalized = [] + for column in columns: + if not isinstance(column, tuple) or not 2 <= len(column) <= 4: + raise TypeError("TVP columns use (name, sql_type[, size_or_precision[, scale]])") + name, sql_type = column[:2] + if not isinstance(name, str) or not name: + raise TypeError("TVP column names must be non-empty strings") + if type(sql_type) is not int: + raise TypeError(f"TVP column {name}: sql_type must be an SQL_* integer constant") + sql_type = {9: 91, 10: 92, 11: 93}.get(sql_type, sql_type) + if sql_type not in self._types: + raise ValueError(f"TVP column {name}: unsupported SQL type {sql_type}") + variable = sql_type in self._variable + numeric = sql_type in (SQL.SQL_DECIMAL.value, SQL.SQL_NUMERIC.value) + temporal = sql_type in (92, 93, -154, -155) + if (variable and len(column) < 3) or (numeric and len(column) != 4): + raise ValueError(f"TVP column {name}: explicit size or precision/scale is required") + size = column[2] if len(column) >= 3 else self._sizes.get(sql_type, 0) + scale = column[3] if len(column) == 4 else (7 if temporal else 0) + if type(size) is not int or type(scale) is not int: + raise TypeError(f"TVP column {name}: size and scale must be integers") + if not 0 <= size <= 2147483647 or scale < 0: + raise ValueError(f"TVP column {name}: invalid size or scale") + if numeric and not (1 <= size <= 38 and 0 <= scale <= size): + raise ValueError( + f"TVP column {name}: require 1 <= precision <= 38 and scale <= precision" + ) + if temporal and scale > 7: + raise ValueError(f"TVP column {name}: temporal scale must be between 0 and 7") + if not numeric and not temporal and scale: + raise ValueError( + f"TVP column {name}: scale is only valid for numeric/temporal types" + ) + if ( + sql_type in (SQL.SQL_CHAR.value, SQL.SQL_WCHAR.value, SQL.SQL_BINARY.value) + and not size + ): + raise ValueError(f"TVP column {name}: fixed-length columns require a positive size") + normalized.append((name, sql_type, size, scale)) + if not isinstance(rows, (list, tuple, Iterator, AsyncIterable)): + raise TypeError("TVP rows must be a list, tuple, or iterator; use [] for an empty TVP") + self._type_name, self._schema = type_name, schema + self._columns, self._rows = tuple(normalized), rows + + @property + def type_name(self): + return self._type_name + + @property + def schema(self): + return self._schema + + @property + def columns(self): + return self._columns + + @property + def rows(self): + return self._rows + + # The existing setinputsizes map owns SQL-to-C mappings. TVP time keeps + # fractional seconds, and decimals use the scalar exact-text conversion. + def _odbc_columns(self): + from .cursor import Cursor + + mapping = Cursor._get_sql_to_c_type_map() + return [ + ( + {-1: 12, -10: -9, -4: -3}.get(sql_type, sql_type), + SQL.SQL_C_WCHAR.value if sql_type in (2, 3, 92, -154) else mapping[sql_type], + size, + scale, + ) + for _, sql_type, size, scale in self.columns + ] + + def _validate_row(self, row, index): + if not isinstance(row, (list, tuple)): + raise TypeError(f"TVP row {index} must be a list or tuple") + if len(row) != len(self.columns): + raise ValueError(f"TVP row {index} has {len(row)} cells; expected {len(self.columns)}") + for (name, sql_type, _, _), value in zip(self.columns, row): + if value is not None and not isinstance(value, self._types[sql_type]): + raise TypeError(f"TVP row {index}, column {name}: incompatible Python value") + if value is None: + continue + if sql_type in self._integer_ranges: + lower, upper = self._integer_ranges[sql_type] + if not lower <= value <= upper: + raise ValueError(f"TVP row {index}, column {name}: integer out of range") + if sql_type in (2, 3) and not value.is_finite(): + raise ValueError(f"TVP row {index}, column {name}: non-finite Decimal") + return row + + _variable = frozenset((1, 12, -1, -8, -9, -10, -2, -3, -4)) + _types = { + **dict.fromkeys((1, 12, -1, -8, -9, -10), str), + **dict.fromkeys((-2, -3, -4), (bytes, bytearray)), + **dict.fromkeys((-6, 5, 4, -5), int), + **dict.fromkeys((6, 7, 8), float), + **dict.fromkeys((2, 3), Decimal), + -7: bool, + 91: date, + 92: time, + -154: time, + 93: datetime, + -155: datetime, + -11: UUID, + } + _sizes = { + -7: 1, + -6: 3, + 5: 5, + 4: 10, + -5: 19, + 6: 15, + 7: 7, + 8: 15, + 91: 10, + 92: 16, + -154: 16, + 93: 27, + -155: 34, + -11: 36, + } + _integer_ranges = { + -6: (0, 255), + 5: (-(2**15), 2**15 - 1), + 4: (-(2**31), 2**31 - 1), + -5: (-(2**63), 2**63 - 1), + } diff --git a/tests/test_004_cursor.py b/tests/test_004_cursor.py index c0afee4d7..001440edf 100644 --- a/tests/test_004_cursor.py +++ b/tests/test_004_cursor.py @@ -1842,7 +1842,6 @@ def test_executemany_unicode_and_empty_strings(cursor, db_connection): (5, ""), (6, "Ñice tëxt"), (7, ""), - (8, "a\x00b"), ] # Execute the batch insert @@ -1854,7 +1853,7 @@ def test_executemany_unicode_and_empty_strings(cursor, db_connection): results = cursor.fetchall() # Check that we got the right number of rows - assert len(results) == 8, f"Expected 8 rows, got {len(results)}" + assert len(results) == 7, f"Expected 7 rows, got {len(results)}" # Check each row for i, (actual, expected_row) in enumerate(zip(results, test_data)): diff --git a/tests/test_004_cursor_timeout.py b/tests/test_004_cursor_timeout.py index d9cc4a8ac..1eeb94eda 100644 --- a/tests/test_004_cursor_timeout.py +++ b/tests/test_004_cursor_timeout.py @@ -1,75 +1,15 @@ -"""Regression tests for query-timeout statement-handle routing.""" - -from unittest.mock import Mock +"""Live regression coverage for query timeout and subsequent cursor recovery.""" import pytest -from mssql_python import ddbc_bindings -from mssql_python.constants import ConstantsDDBC -from mssql_python.cursor import Cursor, logger - - -@pytest.mark.parametrize("metadata_handle", [False, True], ids=["default", "metadata"]) -@pytest.mark.parametrize("timeout", [0, 2], ids=["disabled", "enabled"]) -def test_set_timeout_uses_target_handle(monkeypatch, metadata_handle, timeout): - cursor = Cursor.__new__(Cursor) - cursor.hstmt = object() - cursor._timeout = timeout - statement_handle = object() if metadata_handle else None - target = statement_handle or cursor.hstmt - set_attribute = Mock(return_value=0) - check_error = Mock() - warning = Mock() - monkeypatch.setattr(ddbc_bindings, "DDBCSQLSetStmtAttr", set_attribute) - monkeypatch.setattr("mssql_python.cursor.check_error", check_error) - monkeypatch.setattr(logger, "warning", warning) - - assert cursor._set_timeout(statement_handle) is None - - if timeout: - set_attribute.assert_called_once_with( - target, ConstantsDDBC.SQL_ATTR_QUERY_TIMEOUT.value, timeout - ) - check_error.assert_called_once_with(ConstantsDDBC.SQL_HANDLE_STMT.value, target, 0) - else: - set_attribute.assert_not_called() - check_error.assert_not_called() - warning.assert_not_called() - - -@pytest.mark.parametrize("metadata_handle", [False, True], ids=["default", "metadata"]) -def test_set_timeout_failure_reports_target_diagnostics( - db_connection, monkeypatch, metadata_handle -): - with db_connection.cursor() as cursor: - cursor._timeout = 2 - if metadata_handle: - cursor._tvp_metadata_hstmt = db_connection._conn.alloc_statement_handle() - statement_handle = cursor._tvp_metadata_hstmt - target = statement_handle or cursor.hstmt - real_set_attribute = ddbc_bindings.DDBCSQLSetStmtAttr - real_check_error = ddbc_bindings.DDBCSQLCheckError - - def fail_attribute(handle, attribute, value): - # An invalid identifier produces real driver diagnostics on this handle. - return real_set_attribute(handle, 999999, value) +from mssql_python import OperationalError +from mssql_python.cursor import Cursor - set_attribute = Mock(side_effect=fail_attribute) - check_error = Mock(wraps=real_check_error) - warning = Mock() - with monkeypatch.context() as patch: - patch.setattr(ddbc_bindings, "DDBCSQLSetStmtAttr", set_attribute) - patch.setattr(ddbc_bindings, "DDBCSQLCheckError", check_error) - patch.setattr(logger, "warning", warning) - assert cursor._set_timeout(statement_handle) is None - set_attribute.assert_called_once_with(target, ConstantsDDBC.SQL_ATTR_QUERY_TIMEOUT.value, 2) - check_error.assert_called_once_with(ConstantsDDBC.SQL_HANDLE_STMT.value, target, -1) - assert check_error.call_args.args[1] is target - error_info = real_check_error(ConstantsDDBC.SQL_HANDLE_STMT.value, target, -1) - assert error_info.sqlState == "HY092" - assert error_info.ddbcErrorMsg - warning.assert_called_once() - assert warning.call_args.args[0] == "Failed to set query timeout: %s" - assert "Invalid attribute/option identifier" in warning.call_args.args[1] - assert cursor.execute("SELECT 1").fetchone()[0] == 1 +def test_query_timeout_and_recovery(db_connection): + with Cursor(db_connection, timeout=1) as cursor: + with pytest.raises(OperationalError, match="[Tt]imeout"): + cursor.execute("WAITFOR DELAY '00:00:03'; SELECT 1") + assert cursor.execute("SELECT 42").fetchone()[0] == 42 + with Cursor(db_connection, timeout=0) as cursor: + assert cursor.execute("SELECT 43").fetchone()[0] == 43 diff --git a/tests/test_028_table_valued_parameters.py b/tests/test_028_table_valued_parameters.py index 154de1022..598ebb542 100644 --- a/tests/test_028_table_valued_parameters.py +++ b/tests/test_028_table_valued_parameters.py @@ -1,25 +1,40 @@ -""" -Integration coverage for SQL Server table-valued parameters. -""" +"""Live SQL Server coverage for the shared explicit TVP contract and ODBC streaming.""" +import asyncio import datetime +import inspect import os import subprocess import sys +import threading import uuid -from decimal import Decimal +import weakref +from decimal import Decimal, localcontext import pytest +import pytest_asyncio from mssql_python import ( DatabaseError, NotSupportedError, - ProgrammingError, + TableValuedParameter, + connect, SQL_INTEGER, SQL_WVARCHAR, - ddbc_bindings, + SQL_VARBINARY, + SQL_DECIMAL, + SQL_BIT, + SQL_TINYINT, + SQL_SMALLINT, + SQL_BIGINT, + SQL_DOUBLE, + SQL_TYPE_DATE, + SQL_TYPE_TIME, + SQL_TYPE_TIMESTAMP, + SQL_GUID, ) -from mssql_python.odbc_provider import ProviderManager +from mssql_python.async_query import _AsyncConnection +from mssql_python.constants import ConstantsDDBC from test_023_ssh_tunnel_gil_release import ( WATCHDOG_SECONDS, _parse_server, @@ -29,704 +44,627 @@ @pytest.fixture -def tvp_procedure(cursor, db_connection): +def tvp_schema(cursor, db_connection): suffix = uuid.uuid4().hex - type_name = f"pytest_tvp_type_{suffix}" - wildcard_match_type_name = type_name.replace("_", "X") - procedure_name = f"pytest_tvp_proc_{suffix}" - - try: - cursor.execute( - f"CREATE TYPE dbo.[{type_name}] AS TABLE " "(id int NOT NULL, label nvarchar(100) NULL)" - ) - cursor.execute( - f"CREATE TYPE dbo.[{wildcard_match_type_name}] AS TABLE " "(decoy_id bigint NOT NULL)" - ) - cursor.execute(f""" - CREATE PROCEDURE dbo.[{procedure_name}] - @prefix int, - @items dbo.[{type_name}] READONLY, - @suffix nvarchar(20) - AS - BEGIN - SET NOCOUNT ON; - SELECT id + @prefix, label + @suffix - FROM @items - ORDER BY id; - END - """) - db_connection.commit() - yield type_name, procedure_name - finally: - cursor.execute(f"DROP PROCEDURE IF EXISTS dbo.[{procedure_name}]") - cursor.execute(f"DROP TYPE IF EXISTS dbo.[{type_name}]") - cursor.execute(f"DROP TYPE IF EXISTS dbo.[{wildcard_match_type_name}]") - db_connection.commit() - - -@pytest.fixture -def typed_tvp_procedure(cursor, db_connection): - suffix = uuid.uuid4().hex - type_name = f"pytest_typed_tvp_type_{suffix}" - procedure_name = f"pytest_typed_tvp_proc_{suffix}" - + schema = f"pytest_tvp_{suffix}" + cursor.execute(f"CREATE SCHEMA [{schema}]") + cursor.execute(f"CREATE TYPE [{schema}].Items AS TABLE (id int, label nvarchar(max))") + cursor.execute(f"CREATE TABLE [{schema}].Calls (n int)") + cursor.execute(f""" + CREATE PROCEDURE [{schema}].Echo @prefix int, @items [{schema}].Items READONLY, + @suffix nvarchar(max) + AS BEGIN + SET NOCOUNT ON; + INSERT INTO [{schema}].Calls VALUES (1); + SELECT id + @prefix AS id, label + @suffix AS label FROM @items ORDER BY id; + END + """) + cursor.execute(f""" + CREATE PROCEDURE [{schema}].Pair @left [{schema}].Items READONLY, + @right [{schema}].Items READONLY + AS BEGIN + SET NOCOUNT ON; + SELECT id, label FROM @left UNION ALL SELECT id, label FROM @right; + END + """) + cursor.execute(f""" + CREATE PROCEDURE [{schema}].CountRows @items [{schema}].Items READONLY + AS BEGIN + SET NOCOUNT ON; + INSERT INTO [{schema}].Calls VALUES (1); + SELECT COUNT_BIG(*), SUM(CONVERT(bigint,id)) FROM @items; + END + """) + db_connection.commit() try: - cursor.execute(f""" - CREATE TYPE dbo.[{type_name}] AS TABLE ( - bit_value bit NULL, - tiny_value tinyint NULL, - small_value smallint NULL, - int_value int NULL, - big_value bigint NULL, - float_value float NULL, - decimal_value decimal(20, 6) NULL, - text_value nvarchar(100) NULL, - binary_value varbinary(100) NULL, - date_value date NULL, - time_value time(6) NULL, - datetime_value datetime2(6) NULL, - offset_value datetimeoffset(6) NULL, - guid_value uniqueidentifier NULL, - all_null_binary varbinary(16) NULL - ) - """) - cursor.execute(f""" - CREATE PROCEDURE dbo.[{procedure_name}] - @items dbo.[{type_name}] READONLY - AS - BEGIN - SET NOCOUNT ON; - SELECT * - FROM @items - ORDER BY int_value; - END - """) - db_connection.commit() - yield procedure_name + yield schema finally: - cursor.execute(f"DROP PROCEDURE IF EXISTS dbo.[{procedure_name}]") - cursor.execute(f"DROP TYPE IF EXISTS dbo.[{type_name}]") + cursor.execute(f"DROP PROCEDURE [{schema}].Echo") + cursor.execute(f"DROP PROCEDURE [{schema}].Pair") + cursor.execute(f"DROP PROCEDURE [{schema}].CountRows") + cursor.execute(f"DROP TYPE [{schema}].Items") + cursor.execute(f"DROP TABLE [{schema}].Calls") + cursor.execute(f"DROP SCHEMA [{schema}]") db_connection.commit() -@pytest.fixture -def multiple_tvp_procedure(cursor, db_connection): - suffix = uuid.uuid4().hex - type_name = f"pytest_multi_tvp_type_{suffix}" - procedure_name = f"pytest_multi_tvp_proc_{suffix}" +def items(schema, rows): + return TableValuedParameter( + "Items", [("id", SQL_INTEGER), ("label", SQL_WVARCHAR, 0)], rows, schema=schema + ) - try: - cursor.execute(f"CREATE TYPE dbo.[{type_name}] AS TABLE (id int NOT NULL)") - cursor.execute(f""" - CREATE PROCEDURE dbo.[{procedure_name}] - @left_rows dbo.[{type_name}] READONLY, - @offset int, - @right_rows dbo.[{type_name}] READONLY - AS - BEGIN - SET NOCOUNT ON; - SELECT side, id + @offset - FROM ( - SELECT 'left' AS side, id FROM @left_rows - UNION ALL - SELECT 'right', id FROM @right_rows - ) AS combined - ORDER BY side, id; - END - """) - db_connection.commit() - yield procedure_name - finally: - cursor.execute(f"DROP PROCEDURE IF EXISTS dbo.[{procedure_name}]") - cursor.execute(f"DROP TYPE IF EXISTS dbo.[{type_name}]") - db_connection.commit() +async def call(method, *args, **kwargs): + result = method(*args, **kwargs) + return await result if inspect.isawaitable(result) else result -@pytest.fixture -def schema_qualified_large_tvp(cursor, db_connection): - suffix = uuid.uuid4().hex - schema_name = f"pytest_tvp_schema_{suffix}" - type_name = f"pytest_large_tvp_type_{suffix}" - procedure_name = f"pytest_large_tvp_proc_{suffix}" +@pytest_asyncio.fixture(params=["sync", "async"]) +async def tvp_cursor(request, conn_str): + if request.param == "sync": + connection = connect(conn_str, autocommit=True) + else: + connection = await _AsyncConnection.connect(conn_str, autocommit=True) + cursor = connection.cursor() try: - cursor.execute(f"CREATE SCHEMA [{schema_name}]") - cursor.execute(f""" - CREATE TYPE [{schema_name}].[{type_name}] AS TABLE ( - text_value nvarchar(max) NULL, - binary_value varbinary(max) NULL - ) - """) - cursor.execute(f""" - CREATE PROCEDURE [{schema_name}].[{procedure_name}] - @items [{schema_name}].[{type_name}] READONLY - AS - BEGIN - SET NOCOUNT ON; - SELECT text_value, binary_value FROM @items; - END - """) - db_connection.commit() - yield schema_name, procedure_name + yield cursor finally: - cursor.execute(f"DROP PROCEDURE IF EXISTS [{schema_name}].[{procedure_name}]") - cursor.execute(f"DROP TYPE IF EXISTS [{schema_name}].[{type_name}]") - cursor.execute(f"DROP SCHEMA IF EXISTS [{schema_name}]") - db_connection.commit() + await call(cursor.close) + await call(connection.close) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("rows", [[], [(1, "Keyboard"), (2, None)], ((3, ""),)]) +@pytest.mark.parametrize("named", [False, True]) +async def test_common_rows_named_reuse_and_one_invocation(tvp_cursor, tvp_schema, rows, named): + value = items(tvp_schema, rows) + sql = f"EXEC [{tvp_schema}].Echo ?, ?, ?" + params = (10, value, "✓") + if named: + sql = f"EXEC [{tvp_schema}].Echo %(prefix)s, %(items)s, %(suffix)s" + params = {"prefix": 10, "items": value, "suffix": "✓"} + for _ in range(2): + await call(tvp_cursor.execute, sql, params) + result = await call(tvp_cursor.fetchall) + assert [tuple(row) for row in result] == [ + (row[0] + 10, None if row[1] is None else row[1] + "✓") for row in rows + ] + await call(tvp_cursor.execute, f"SELECT COUNT(*) FROM [{tvp_schema}].Calls") + assert (await call(tvp_cursor.fetchone))[0] == 2 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("left,right", [([], []), ([], [(2, "right")]), ([(1, None)], [(2, "")])]) +async def test_common_multiple_tvps(tvp_cursor, tvp_schema, left, right): + await call( + tvp_cursor.execute, + f"EXEC [{tvp_schema}].Pair ?, ?", + items(tvp_schema, left), + items(tvp_schema, right), + ) + result = await call(tvp_cursor.fetchall) + assert sorted(tuple(row) for row in result) == sorted(left + right) + + +@pytest.mark.asyncio +async def test_common_null_scalars_with_tvp(tvp_cursor, tvp_schema): + await call( + tvp_cursor.execute, + f"EXEC [{tvp_schema}].Echo ?, ?, ?", + None, + items(tvp_schema, [(1, "one")]), + None, + ) + assert tuple(await call(tvp_cursor.fetchone)) == (None, None) -def test_tvp_positional_named_empty_and_statement_reuse(cursor, tvp_procedure): - _, procedure_name = tvp_procedure - sql = f"EXEC dbo.[{procedure_name}] ?, ?, ?" +def test_explicit_type_in_parameterized_batch(cursor, tvp_schema): + cursor.execute("SELECT id, label FROM ?", items(tvp_schema, iter([(1, "one")]))) + assert tuple(cursor.fetchone()) == (1, "one") - cursor.execute(sql, (10, [(1, "Keyboard"), (2, None)], "!")) - assert [tuple(row) for row in cursor.fetchall()] == [(11, "Keyboard!"), (12, None)] - cursor.execute(sql, (10, [], "!")) - assert cursor.fetchall() == [] +def test_reuse_with_empty_and_changing_sources(cursor, tvp_schema): + sql = f"EXEC [{tvp_schema}].Echo ?, ?, ?" + for rows in ([], [(1, None)], [(2, "two"), (3, "")], []): + value = TableValuedParameter( + "Items", + [("diagnostic_id", SQL_INTEGER), ("diagnostic_label", SQL_WVARCHAR, 100)], + iter(rows), + schema=tvp_schema, + ) + cursor.execute(sql, 0, value, "") + assert [tuple(row) for row in cursor.fetchall()] == rows - cursor.execute(sql, (None, [(3, "Nullable")], "")) - assert tuple(cursor.fetchone()) == (None, "Nullable") - cursor.execute( - f"EXEC dbo.[{procedure_name}] %(prefix)s, %(items)s, %(suffix)s", - {"prefix": 0, "items": [(4, "Mouse")], "suffix": "?"}, +def test_declared_text_size_failure_recovers(cursor, tvp_schema): + value = TableValuedParameter( + "Items", + [("id", SQL_INTEGER), ("label", SQL_WVARCHAR, 3)], + iter([(1, "too long")]), + schema=tvp_schema, ) - assert tuple(cursor.fetchone()) == (4, "Mouse?") - - -def test_tvp_empty_first_and_all_null_column(cursor, tvp_procedure): - _, procedure_name = tvp_procedure - sql = f"EXEC dbo.[{procedure_name}] ?, ?, ?" + with pytest.raises((DatabaseError, ValueError)): + cursor.execute(f"EXEC [{tvp_schema}].CountRows ?", value) + assert cursor.execute("SELECT 42").fetchone()[0] == 42 - cursor.execute(sql, (0, [], "")) - assert cursor.fetchall() == [] - cursor.execute(sql, (0, [(1, None), (2, None)], "")) - assert [tuple(row) for row in cursor.fetchall()] == [(1, None), (2, None)] - - -def test_tvp_binds_supported_cell_types(cursor, typed_tvp_procedure): - first_guid = uuid.uuid4() - second_guid = uuid.uuid4() - offset = datetime.timezone(datetime.timedelta(hours=5, minutes=30)) - rows = [ - ( - True, - 255, - -32768, - -2147483648, - -(2**63), - 1.25, - Decimal("-12345678901234.567890"), - "Grüße 😀", - b"\x00\xff", - datetime.date(2026, 9, 15), - datetime.time(12, 34, 56, 123456), - datetime.datetime(2026, 9, 15, 12, 34, 56, 123456), - datetime.datetime(2026, 9, 15, 12, 34, 56, 123456, tzinfo=offset), - first_guid, - None, - ), - ( - False, - 0, - 32767, - 2147483647, - 2**63 - 1, - -2.5, - Decimal("12.345"), - "later row is longer", - b"\x01\x02\x03", - datetime.date(2026, 9, 16), - datetime.time(1, 2, 3, 4), - datetime.datetime(2026, 9, 16, 1, 2, 3, 4), - datetime.datetime(2026, 9, 16, 1, 2, 3, 4, tzinfo=datetime.timezone.utc), - second_guid, - None, - ), +@pytest.mark.asyncio +async def test_common_typed_cells(tvp_cursor, cursor, db_connection): + suffix = uuid.uuid4().hex + name, proc = f"Typed_{suffix}", f"TypedProc_{suffix}" + declarations = [ + ("bit_value", SQL_BIT, "bit"), + ("tiny_value", SQL_TINYINT, "tinyint"), + ("small_value", SQL_SMALLINT, "smallint"), + ("int_value", SQL_INTEGER, "int"), + ("big_value", SQL_BIGINT, "bigint"), + ("float_value", SQL_DOUBLE, "float"), + ("decimal_value", SQL_DECIMAL, "decimal(18,10)", 18, 10), + ("text_value", SQL_WVARCHAR, "nvarchar(max)", 0), + ("binary_value", SQL_VARBINARY, "varbinary(max)", 0), + ("date_value", SQL_TYPE_DATE, "date"), + ("time_value", SQL_TYPE_TIME, "time(7)"), + ("datetime_value", SQL_TYPE_TIMESTAMP, "datetime2(7)"), + ("offset_value", ConstantsDDBC.SQL_DATETIMEOFFSET.value, "datetimeoffset(7)"), + ("guid_value", SQL_GUID, "uniqueidentifier"), + ("all_null_binary", SQL_VARBINARY, "varbinary(16)", 16), ] - - cursor.execute(f"EXEC dbo.[{typed_tvp_procedure}] ?", (rows,)) - actual = [tuple(row) for row in cursor.fetchall()] - - assert actual[0][:9] == rows[0][:9] - assert actual[1][:9] == rows[1][:9] - assert actual[0][9:13] == rows[0][9:13] - assert actual[1][9:13] == rows[1][9:13] - assert str(actual[0][13]) == str(first_guid) - assert str(actual[1][13]) == str(second_guid) - assert actual[0][14] is None - assert actual[1][14] is None - - invalid_row = list(rows[0]) - invalid_row[6] = Decimal("NaN") - with pytest.raises(ValueError, match="non-finite Decimal"): - cursor.execute( - f"EXEC dbo.[{typed_tvp_procedure}] ?", - ([tuple(invalid_row)],), - ) - - -def test_multiple_tvps_with_interleaved_scalar(cursor, multiple_tvp_procedure): + columns = [(d[0], d[1], *d[3:]) for d in declarations] + row = ( + True, + 255, + -32768, + 2147483647, + -(2**63), + 1.25, + Decimal("12345678.1234567890"), + "long🚀漢字\x00" * 2000, + b"\x00\xff" * 10000, + datetime.date(2020, 2, 29), + datetime.time(1, 2, 3, 123456), + datetime.datetime(2021, 3, 4, 1, 2, 3, 123456), + datetime.datetime( + 2021, + 3, + 4, + 1, + 2, + 3, + 123456, + tzinfo=datetime.timezone(datetime.timedelta(hours=-7, minutes=-30)), + ), + uuid.uuid4(), + None, + ) cursor.execute( - f"EXEC dbo.[{multiple_tvp_procedure}] ?, ?, ?", - ([(1,), (2,)], 10, [(3,), (4,)]), + f"CREATE TYPE dbo.[{name}] AS TABLE (" + + ",".join(f"[{d[0]}] {d[2]} NULL" for d in declarations) + + ")" ) - assert [tuple(row) for row in cursor.fetchall()] == [ - ("left", 11), - ("left", 12), - ("right", 13), - ("right", 14), - ] - + projection = ",".join( + "CONVERT(varbinary(max),text_value) AS text_value" if d[0] == "text_value" else d[0] + for d in declarations + ) + cursor.execute( + f"CREATE PROCEDURE dbo.[{proc}] @items dbo.[{name}] READONLY AS " + f"SELECT {projection} FROM @items" + ) + db_connection.commit() + try: + with localcontext() as ctx: + ctx.prec = 5 + source = [list(row), [None] * len(row)] + value = TableValuedParameter(f"dbo.{name}", columns, source) + await call(tvp_cursor.execute, f"EXEC dbo.[{proc}] ?", value) + results = [tuple(r) for r in await call(tvp_cursor.fetchall)] + assert len(results) == 2 + actual = next(r for r in results if r[0] is not None) + expected = list(row) + # Verify every UTF-16 byte stored by SQL Server, including embedded NULs, + # independently of the result decoder's text-terminator handling. + expected[7] = row[7].encode("utf-16le") + assert actual == tuple(expected) + assert (None,) * len(row) in results + assert source == [list(row), [None] * len(row)] + finally: + cursor.execute(f"DROP PROCEDURE dbo.[{proc}]") + cursor.execute(f"DROP TYPE dbo.[{name}]") + db_connection.commit() -def test_schema_qualified_tvp_with_large_and_embedded_null_values( - cursor, schema_qualified_large_tvp -): - schema_name, procedure_name = schema_qualified_large_tvp - large_text = "prefix\x00suffix" + "😀" * 2500 - large_binary = b"\x00\xff" * 4501 - rows = ((large_text, large_binary), (None, None)) - cursor.execute(f"EXEC [{schema_name}].[{procedure_name}] ?", (rows,)) - actual = [tuple(row) for row in cursor.fetchall()] +@pytest.mark.asyncio +async def test_common_date_range(tvp_cursor, cursor, db_connection): + name = f"Dates_{uuid.uuid4().hex}" + rows = [ + (None,), + (datetime.date.min,), + (datetime.date(1752, 12, 31),), + (datetime.date(1753, 1, 1),), + (datetime.date.max,), + ] + cursor.execute(f"CREATE TYPE dbo.[{name}] AS TABLE (value date NULL)") + db_connection.commit() + try: + value = TableValuedParameter(f"dbo.{name}", [("value", SQL_TYPE_DATE)], rows) + await call(tvp_cursor.execute, "SELECT value FROM ? ORDER BY value", value) + assert [tuple(row) for row in await call(tvp_cursor.fetchall)] == rows + finally: + cursor.execute(f"DROP TYPE dbo.[{name}]") + db_connection.commit() - assert actual == [(large_text, large_binary), (None, None)] +def test_generator_is_lazy_bounded_and_invokes_once(cursor, tvp_schema): + seen = [] + live = weakref.WeakSet() + peak = 0 + + class LiveRow(list): + __hash__ = object.__hash__ + + def rows(): + nonlocal peak + for i in range(5000): + row = LiveRow((i, "x" * 1000)) + live.add(row) + peak = max(peak, len(live)) + assert len(live) <= 2, "The driver retained earlier generator rows" + seen.append(i) + yield row + + value = items(tvp_schema, rows()) + assert seen == [] + cursor.execute(f"EXEC [{tvp_schema}].CountRows ?", value) + assert tuple(cursor.fetchone()) == (5000, 4999 * 5000 // 2) + assert len(seen) == 5000 + assert peak <= 2 + assert len(live) == 0 + cursor.execute(f"SELECT COUNT(*) FROM [{tvp_schema}].Calls") + assert cursor.fetchone()[0] == 1 + + +@pytest.mark.parametrize("failure", [ValueError("source failed"), KeyboardInterrupt("interrupted")]) +def test_generator_failure_preserves_error_and_recovers(cursor, tvp_schema, failure): + def rows(): + yield 1, "first" + raise failure + + with pytest.raises(type(failure)) as error: + cursor.execute(f"EXEC [{tvp_schema}].Echo ?, ?, ?", 0, items(tvp_schema, rows()), "") + assert error.value is failure + assert cursor.execute("SELECT 42").fetchone()[0] == 42 + cursor.execute(f"SELECT COUNT(*) FROM [{tvp_schema}].Calls") + assert cursor.fetchone()[0] == 0 + cursor.execute(f"EXEC [{tvp_schema}].Echo ?, ?, ?", 0, items(tvp_schema, [(2, "ok")]), "") + assert tuple(cursor.fetchone()) == (2, "ok") -def test_failed_tvp_does_not_poison_cursor(cursor, tvp_procedure): - _, procedure_name = tvp_procedure - sql = f"EXEC dbo.[{procedure_name}] ?, ?, ?" - with pytest.raises(DatabaseError): - cursor.execute(sql, (0, [(None, "invalid")], "")) +@pytest.mark.parametrize("rows", [[], [(1, None)], [(1, ""), (2, "🚀" * 10000)]]) +def test_streaming_empty_null_and_large_cells(cursor, tvp_schema, rows): + cursor.execute(f"EXEC [{tvp_schema}].Echo ?, ?, ?", 0, items(tvp_schema, iter(rows)), "") + assert [tuple(row) for row in cursor.fetchall()] == rows - cursor.execute("SELECT ?", 42) - assert cursor.fetchone()[0] == 42 - cursor.execute(sql, (0, [(1, "valid")], "")) - assert tuple(cursor.fetchone()) == (1, "valid") +@pytest.mark.parametrize("bad", [[(1,)], [(1, "a", "extra")], [1], [(1, object())]]) +def test_bad_stream_row_recovers(cursor, tvp_schema, bad): + with pytest.raises((TypeError, ValueError)): + cursor.execute(f"EXEC [{tvp_schema}].CountRows ?", items(tvp_schema, iter(bad))) + assert cursor.execute("SELECT 1").fetchone()[0] == 1 -def test_tvp_rejects_invalid_shape_and_cells(cursor, tvp_procedure): - _, procedure_name = tvp_procedure - sql = f"EXEC dbo.[{procedure_name}] ?, ?, ?" +def test_implicit_nested_sequences_are_not_tvps(cursor): + for value in ([], [(1, "a")], ((1, "a"),)): + with pytest.raises(TypeError, match="Unsupported parameter type"): + cursor.execute("SELECT ?", (value,)) + assert cursor.execute("SELECT 1").fetchone()[0] == 1 - with pytest.raises(ValueError, match="same number"): - cursor.execute(sql, (0, [(1, "valid"), (2,)], "")) - with pytest.raises(TypeError, match="incompatible Python types"): - cursor.execute(sql, (0, [(1, "valid"), ("2", "invalid")], "")) - with pytest.raises(TypeError, match="only be bound"): - cursor.execute("SELECT CAST(? AS int)", ([(1,)],)) +@pytest.mark.asyncio +async def test_async_rejects_streaming_before_consumption(conn_str, tvp_schema): + seen = [] -def test_tvp_rejected_by_executemany_before_first_write(cursor, tvp_procedure): - cursor.execute("CREATE TABLE #pytest_tvp_executemany (id int)") + def rows(): + seen.append(1) + yield 1, "never" - with pytest.raises(NotSupportedError): - cursor.executemany( - "INSERT INTO #pytest_tvp_executemany VALUES (?)", - [(1,), ([(2, "label")],)], - ) + async def async_rows(): + seen.append(1) + yield 1, "never" - cursor.execute("SELECT COUNT(*) FROM #pytest_tvp_executemany") - assert cursor.fetchone()[0] == 0 - - -def test_tvp_rejected_with_setinputsizes(cursor, tvp_procedure): - _, procedure_name = tvp_procedure - cursor.setinputsizes( - [ - (SQL_INTEGER, 10, 0), - (SQL_WVARCHAR, 100, 0), - (SQL_WVARCHAR, 20, 0), - ] - ) + connection = await _AsyncConnection.connect(conn_str, autocommit=True) + cursor = connection.cursor() try: - with pytest.raises(NotSupportedError): - cursor.execute( - f"EXEC dbo.[{procedure_name}] ?, ?, ?", - (0, [(1, "label")], ""), - ) + for source in (rows(), async_rows()): + with pytest.raises(NotSupportedError, match="materialized"): + await cursor.execute(f"EXEC [{tvp_schema}].CountRows ?", items(tvp_schema, source)) + assert seen == [] + await cursor.execute("SELECT 42") + assert (await cursor.fetchone())[0] == 42 finally: - cursor.setinputsizes(None) - + await cursor.close() + await connection.close() -def test_tvp_rejected_by_mssql_odbc_provider(cursor, tvp_procedure, monkeypatch): - _, procedure_name = tvp_procedure - monkeypatch.setattr(ProviderManager, "effective", lambda *_: "mssql-odbc") - with pytest.raises(NotSupportedError, match="mssql-odbc provider"): - cursor.execute( - f"EXEC dbo.[{procedure_name}] ?, ?, ?", - (0, [(1, "label")], ""), - ) +@pytest.mark.parametrize( + "kwargs,error", + [ + ({"rows": None}, TypeError), + ({"columns": []}, TypeError), + ({"columns": [("x", SQL_WVARCHAR)]}, ValueError), + ({"columns": [("x", SQL_DECIMAL, 39, 2)]}, ValueError), + ({"columns": [("x", SQL_DECIMAL, 4, 5)]}, ValueError), + ({"columns": [("x", 9999)]}, ValueError), + ({"type_name": "db.dbo.Items"}, ValueError), + ({"type_name": "dbo.Items", "schema": "dbo"}, ValueError), + ], +) +def test_explicit_contract_validation(kwargs, error): + arguments = {"type_name": "Items", "columns": [("id", SQL_INTEGER)], "rows": []} + arguments.update(kwargs) + with pytest.raises(error): + TableValuedParameter(**arguments) -def test_tvp_metadata_handle_inherits_query_timeout(db_connection, tvp_procedure, monkeypatch): - _, procedure_name = tvp_procedure - real_set_stmt_attr = ddbc_bindings.DDBCSQLSetStmtAttr - timeout_handles = [] +@pytest.mark.parametrize("bad", [2**31, -(2**31) - 1]) +def test_explicit_integer_width_does_not_truncate(cursor, tvp_schema, bad): + with pytest.raises(ValueError, match="out of range"): + cursor.execute(f"EXEC [{tvp_schema}].CountRows ?", items(tvp_schema, [(bad, None)])) + assert cursor.execute("SELECT 1").fetchone()[0] == 1 - def record_timeout(statement_handle, attribute, value): - timeout_handles.append(statement_handle) - return real_set_stmt_attr(statement_handle, attribute, value) - db_connection.timeout = 2 - monkeypatch.setattr( - ddbc_bindings, - "DDBCSQLSetStmtAttr", - record_timeout, +def test_precise_decimal_streaming(cursor, db_connection): + name = f"Decimal_{uuid.uuid4().hex}" + proc = f"Proc_{name}" + cursor.execute(f"CREATE TYPE dbo.[{name}] AS TABLE (amount decimal(38,6))") + cursor.execute( + f"CREATE PROC dbo.[{proc}] @items dbo.[{name}] READONLY AS " "SELECT amount FROM @items" ) + db_connection.commit() + values = [ + Decimal("12345678901234567890123456789012.123456"), + None, + Decimal("0.000001"), + Decimal("-99999999999999999999999999999999.999999"), + ] try: - with db_connection.cursor() as timeout_cursor: - timeout_cursor.execute( - f"EXEC dbo.[{procedure_name}] ?, ?, ?", - (0, [(1, "label")], ""), + with localcontext() as context: + context.prec = 3 + value = TableValuedParameter( + name, [("amount", SQL_DECIMAL, 38, 6)], ((v,) for v in values) ) - assert tuple(timeout_cursor.fetchone()) == (1, "label") + cursor.execute(f"EXEC dbo.[{proc}] ?", value) + assert [r[0] for r in cursor.fetchall()] == values + for value in (Decimal("NaN"), Decimal("Infinity")): + with pytest.raises(ValueError, match="non-finite"): + cursor.execute( + f"EXEC dbo.[{proc}] ?", + TableValuedParameter(name, [("amount", SQL_DECIMAL, 38, 6)], [(value,)]), + ) + assert cursor.execute("SELECT 1").fetchone()[0] == 1 finally: - db_connection.timeout = 0 - - assert len(timeout_handles) == 2 - assert timeout_handles[0] is not timeout_handles[1] + cursor.execute(f"DROP PROC dbo.[{proc}]") + cursor.execute(f"DROP TYPE dbo.[{name}]") + db_connection.commit() -@pytest.fixture -def conversion_tvp(cursor, db_connection, request): - suffix = uuid.uuid4().hex - type_name = f"pytest_conversion_type_{suffix}" - procedure_name = f"pytest_conversion_proc_{suffix}" +def test_sync_temporal_scale_six(cursor, db_connection): + name = f"Times_{uuid.uuid4().hex}" + proc = f"Proc_{name}" + cursor.execute(f"CREATE TYPE dbo.[{name}] AS TABLE (t time(6), d datetime2(6))") + cursor.execute(f"CREATE PROC dbo.[{proc}] @items dbo.[{name}] READONLY AS SELECT * FROM @items") + db_connection.commit() + rows = [ + (datetime.time(1, 2, 3, 123456), datetime.datetime(2024, 2, 29, 1, 2, 3, 123456)), + (None, None), + ] try: - cursor.execute( - f"CREATE TYPE dbo.[{type_name}] AS TABLE (position int, value {request.param} NULL)" + value = TableValuedParameter( + name, [("t", SQL_TYPE_TIME, 16, 6), ("d", SQL_TYPE_TIMESTAMP, 26, 6)], iter(rows) ) - cursor.execute( - f"CREATE PROCEDURE dbo.[{procedure_name}] @items dbo.[{type_name}] READONLY AS " - "SET NOCOUNT ON; SELECT value FROM @items ORDER BY position" - ) - db_connection.commit() - yield f"EXEC dbo.[{procedure_name}] ?" + cursor.execute(f"EXEC dbo.[{proc}] ?", value) + assert [tuple(row) for row in cursor.fetchall()] == rows finally: - cursor.execute(f"DROP PROCEDURE IF EXISTS dbo.[{procedure_name}]") - cursor.execute(f"DROP TYPE IF EXISTS dbo.[{type_name}]") + cursor.execute(f"DROP PROC dbo.[{proc}]") + cursor.execute(f"DROP TYPE dbo.[{name}]") db_connection.commit() +@pytest.mark.asyncio @pytest.mark.parametrize( - ("conversion_tvp", "values", "expected"), - [ - ( - "float", - [Decimal("1.5"), Decimal("123.45"), None, Decimal("1.50"), Decimal("-0.125")], - [1.5, 123.45, None, 1.5, -0.125], - ), - ("int", [Decimal("1.5"), Decimal("-123.45"), None], [1, -123, None]), - ("nvarchar(40)", [Decimal("1.5"), None], ["1.5", None]), - ( - "nvarchar(40)", - [Decimal("1.5"), Decimal("123.45"), None, Decimal("-0.125")], - ["1.500", "123.450", None, "-0.125"], - ), - ( - "decimal(5,2)", - [Decimal("1.239"), None, Decimal("-1.235"), Decimal("9.999")], - [Decimal("1.24"), None, Decimal("-1.24"), Decimal("10.00")], - ), - ( - "decimal(8,2)", - ["000000000000000001.23", None, "-0000000000000001.239"], - [Decimal("1.23"), None, Decimal("-1.24")], - ), - ( - "datetime", - [datetime.datetime(2026, 9, 15, 12, 34, 56, 456789), None], - [datetime.datetime(2026, 9, 15, 12, 34, 56, 457000), None], - ), - ( - "datetime2(3)", - [datetime.datetime(2026, 9, 15, 23, 59, 59, 999999), None], - [datetime.datetime(2026, 9, 16), None], - ), - ( - "time(3)", - [datetime.time(12, 34, 56, 456789), None], - [datetime.time(12, 34, 56, 457000), None], - ), - ( - "datetimeoffset(3)", - [ - datetime.datetime(2026, 9, 15, 23, 59, 59, 999999, tzinfo=datetime.timezone.utc), - None, - ], - [datetime.datetime(2026, 9, 16, tzinfo=datetime.timezone.utc), None], - ), - ], - indirect=["conversion_tvp"], -) -def test_tvp_source_precision_survives_destination_conversion( - cursor, conversion_tvp, values, expected -): - rows = list(enumerate(values)) - for _ in range(2): - cursor.execute(conversion_tvp, (rows,)) - assert [row[0] for row in cursor.fetchall()] == expected - assert rows == list(enumerate(values)) - - -@pytest.mark.parametrize( - ("conversion_tvp", "value", "recovery"), - [ - ("nvarchar(3)", "x" * 5000, "ok"), - ("nvarchar(3)", "\u0100" * 5000, "ok"), - ("varchar(8000)", "x" * 8001, "ok"), - ("varchar(8000) COLLATE Latin1_General_100_CI_AS", "\u00e9" * 8001, "ok"), - ("varbinary(8000)", b"x" * 8001, b"ok"), - ], - ids=["short-ascii", "short-unicode", "long-ascii", "long-unicode", "long-binary"], - indirect=["conversion_tvp"], -) -def test_tvp_destination_limits_leave_cursor_reusable(cursor, conversion_tvp, value, recovery): - with pytest.raises(ProgrammingError, match="would be truncated") as raised: - cursor.execute(conversion_tvp, ([(0, value)],)) - assert raised.value.driver_error == "Syntax error or access violation" - cursor.execute("SELECT ?", 42) - assert cursor.fetchone()[0] == 42 - cursor.execute(conversion_tvp, ([(0, recovery)],)) - assert cursor.fetchone()[0] == recovery - - -@pytest.mark.parametrize("conversion_tvp", ["decimal(38,0)"], indirect=True) -@pytest.mark.parametrize( - "value", [Decimal("NaN"), Decimal("Infinity"), Decimal("1E+39"), Decimal("1E-39")] -) -def test_tvp_decimal_validation_matches_scalar(cursor, conversion_tvp, value): - with pytest.raises(ValueError): - cursor.execute(conversion_tvp, ([(0, value)],)) - cursor.execute(conversion_tvp, ([(0, Decimal("1E+37")), (1, None)],)) - assert [row[0] for row in cursor.fetchall()] == [Decimal("1E+37"), None] - - -@pytest.mark.parametrize("conversion_tvp", ["bigint"], indirect=True) -def test_tvp_integer_widening_and_nulls(cursor, conversion_tvp): - edges = [0, 255, -32768, 32767, -(2**31), 2**31 - 1, -(2**63), 2**63 - 1] - for ordered in (edges, list(reversed(edges))): - values = [None, *ordered, None] - cursor.execute(conversion_tvp, (list(enumerate(values)),)) - assert [row[0] for row in cursor.fetchall()] == values - - -@pytest.mark.parametrize( - ("conversion_tvp", "values"), + "columns,error", [ - ("nvarchar(max)", [None, "ascii first", "\U0001f600" * 4001, "a\x00b", ""]), - ("nvarchar(max)", ["\U0001f600", "later ascii", None]), - ("varchar(max)", [None, "x" * 8001, "a\x00b", ""]), - ("varbinary(max)", [b"", None, b"\x00\xff" * 4501, bytearray(b"last")]), + ([("amount", SQL_DECIMAL, 20, 6)], "precision 18 and scale 10"), + ([("stamp", SQL_TYPE_TIMESTAMP, 26, 6)], "scale 7"), ], - indirect=["conversion_tvp"], ) -def test_tvp_source_buffers_cover_every_row(cursor, conversion_tvp, values): - cursor.execute(conversion_tvp, (list(enumerate(values)),)) - assert [row[0] for row in cursor.fetchall()] == values - - -@pytest.mark.parametrize( - "conversion_tvp", ["varchar(8000) COLLATE Latin1_General_100_CI_AS"], indirect=True -) -@pytest.mark.parametrize("length", [4000, 4001, 8000]) -def test_tvp_unicode_source_fits_bounded_varchar(cursor, conversion_tvp, length): - values = [None, "ascii first", "\u00e9" * length, ""] - for ordered in (values, list(reversed(values))): - cursor.execute(conversion_tvp, (list(enumerate(ordered)),)) - assert [row[0] for row in cursor.fetchall()] == ordered - - -def test_tvp_all_null_columns_use_declared_types(cursor, typed_tvp_procedure): - cursor.execute(f"EXEC dbo.[{typed_tvp_procedure}] ?", ([tuple([None] * 15)],)) - assert tuple(cursor.fetchone()) == tuple([None] * 15) - - -@pytest.mark.parametrize("conversion_tvp", ["float"], indirect=True) -def test_tvp_rejects_unrepresentable_common_decimal_precision(cursor, conversion_tvp): - with pytest.raises(ValueError, match="precision greater than 38"): - cursor.execute(conversion_tvp, ([(0, Decimal("1E+37")), (1, Decimal("0.1"))],)) - cursor.execute(conversion_tvp, ([(0, Decimal("0.1"))],)) - assert cursor.fetchone()[0] == 0.1 - - -@pytest.mark.parametrize("conversion_tvp", ["decimal(5,2)"], indirect=True) -def test_tvp_decimal_wire_format_is_independent_of_text_encoding(conn_str, conversion_tvp): - from mssql_python import SQL_CHAR, connect +async def test_async_native_metadata_limit_is_not_hidden(conn_str, columns, error): + connection = await _AsyncConnection.connect(conn_str, autocommit=True) + cursor = connection.cursor() + try: + await cursor.execute("SELECT 42") + with pytest.raises(ValueError, match=error): + await cursor.execute("SELECT ?", TableValuedParameter("UnusedType", columns, [])) + assert (await cursor.fetchone())[0] == 42 + finally: + await cursor.close() + await connection.close() - with connect(conn_str) as connection: - connection.setencoding("utf-16le", ctype=SQL_CHAR) - with connection.cursor() as encoded_cursor: - encoded_cursor.execute(conversion_tvp, ([(0, Decimal("-1.239")), (1, None)],)) - assert [row[0] for row in encoded_cursor.fetchall()] == [Decimal("-1.24"), None] +def test_schema_mismatch_and_server_error_recovery(cursor, tvp_schema): + with pytest.raises(DatabaseError): + cursor.execute( + f"EXEC [{tvp_schema}].CountRows ?", + TableValuedParameter("DoesNotExist", [("id", SQL_INTEGER)], [], schema=tvp_schema), + ) + assert cursor.execute("SELECT 42").fetchone()[0] == 42 + with pytest.raises(DatabaseError): + cursor.execute( + f"EXEC [{tvp_schema}].CountRows ?", + TableValuedParameter( + "Items", [("only_column", SQL_INTEGER)], [(1,)], schema=tvp_schema + ), + ) + assert cursor.execute("SELECT 43").fetchone()[0] == 43 -@pytest.mark.parametrize("conversion_tvp", ["nvarchar(40)"], indirect=True) -@pytest.mark.parametrize( - ("input_sizes", "error", "message"), - [ - (None, RuntimeError, "metadata requires a valid statement handle"), - ([(SQL_INTEGER, 10, 0)], TypeError, "setinputsizes cannot override"), - ], -) -def test_tvp_native_entry_point_validates_metadata_and_overrides( - db_connection, conversion_tvp, input_sizes, error, message -): - with db_connection.cursor() as native_cursor: - with pytest.raises(error, match=message): - ddbc_bindings.DDBCSQLExecute( - native_cursor.hstmt, - None, - conversion_tvp, - [[(0, "ok")]], - input_sizes, - [False], - True, - {}, - ) - native_cursor.execute(conversion_tvp, ([(0, "ok")],)) - assert native_cursor.fetchone()[0] == "ok" +def test_multiple_generator_tvps_and_large_scalar(cursor, tvp_schema): + cursor.execute( + f"EXEC [{tvp_schema}].Pair ?, ?", + items(tvp_schema, iter([(1, "left"), (2, None)])), + items(tvp_schema, iter([(3, "right")])), + ) + assert sorted(tuple(row) for row in cursor.fetchall()) == [(1, "left"), (2, None), (3, "right")] + suffix = "🚀" * 8000 + cursor.execute( + f"EXEC [{tvp_schema}].Echo ?, ?, ?", 0, items(tvp_schema, iter([(1, "head")])), suffix + ) + assert tuple(cursor.fetchone()) == (1, "head" + suffix) -@pytest.mark.parametrize( - ("rows", "error", "message"), - [ - ([1], TypeError, "rows must be list or tuple"), - ([()], ValueError, "at least one column"), - ([(1,)], ValueError, "declared table type requires"), - ([(1, "ok"), 2], TypeError, "rows must be list or tuple"), - ([(1, [2])], TypeError, "nested row sequences"), - ([(1, Decimal("1")), (2, "2")], TypeError, "incompatible Python types"), - ], -) -def test_tvp_shape_and_type_errors_leave_cursor_reusable( - cursor, tvp_procedure, rows, error, message -): - _, procedure_name = tvp_procedure - sql = f"EXEC dbo.[{procedure_name}] ?, ?, ?" - with pytest.raises(error, match=message): - cursor.execute(sql, (0, rows, "")) - cursor.execute(sql, (0, [(1, "ok")], "")) - assert tuple(cursor.fetchone()) == (1, "ok") +def test_executemany_and_inputsizes_reject_tvps_before_consumption(cursor, tvp_schema): + def rows(): + pytest.fail("Unsupported operation consumed a TVP row source") + yield 1, "never" -@pytest.fixture -def wide_tvp_procedure(cursor, db_connection): - suffix = uuid.uuid4().hex - type_name = f"pytest_wide_tvp_type_{suffix}" - procedure_name = f"pytest_wide_tvp_proc_{suffix}" - # Long column names make metadata span multiple network packets. - columns = ", ".join(["id int"] + [f"c{i}_{'x' * 100} int" for i in range(1, 1024)]) + value = items(tvp_schema, rows()) + with pytest.raises(NotSupportedError, match="executemany"): + cursor.executemany(f"EXEC [{tvp_schema}].CountRows ?", [(value,)]) + cursor.setinputsizes([SQL_INTEGER]) try: - cursor.execute(f"CREATE TYPE dbo.[{type_name}] AS TABLE ({columns})") - cursor.execute( - f"CREATE PROCEDURE dbo.[{procedure_name}] @items dbo.[{type_name}] READONLY AS " - "SET NOCOUNT ON; SELECT id FROM @items ORDER BY id" - ) - db_connection.commit() - yield procedure_name + with pytest.raises(NotSupportedError, match="setinputsizes"): + cursor.execute(f"EXEC [{tvp_schema}].CountRows ?", value) finally: - cursor.execute(f"DROP PROCEDURE IF EXISTS dbo.[{procedure_name}]") - cursor.execute(f"DROP TYPE IF EXISTS dbo.[{type_name}]") - db_connection.commit() - - -def _run_forwarded_tvp(): - import mssql_python - - base = os.environ["DB_CONNECTION_STRING"] - target = _parse_server(base) - assert target is not None, "Could not parse Server=host,port" - host, port = _start_forwarder(target) - mssql_python.pooling(enabled=False) - with mssql_python.connect(_replace_server(base, host, port)) as connection: - with connection.cursor() as cursor: - sql = f"EXEC dbo.[{os.environ['TVP_GIL_PROCEDURE']}] ?" - padding = (None,) * 1023 - if os.environ["TVP_GIL_METADATA_ERROR"] == "1": - import ctypes - - library = ctypes.CDLL(sys.modules["ddbc_bindings"].__file__) - slot = ctypes.c_void_p.in_dll(library, "SQLGetData_ptr") - original = slot.value - get_data_type = ctypes.CFUNCTYPE( - ctypes.c_short, - ctypes.c_void_p, - ctypes.c_ushort, - ctypes.c_short, - ctypes.c_void_p, - ctypes.c_ssize_t, - ctypes.POINTER(ctypes.c_ssize_t), - ) - get_data = get_data_type(original) - - @get_data_type - def invalid_metadata_column(handle, column, ctype, value, size, indicator): - # Produce a real driver error before metadata is drained. Do not - # wrap SQLFreeStmt: its original GIL behavior is what we exercise. - slot.value = original - return get_data(handle, 999, ctype, value, size, indicator) + cursor.setinputsizes([]) + assert cursor.execute("SELECT 1").fetchone()[0] == 1 + + +@pytest.mark.parametrize("operation", ["execute", "close"]) +def test_generator_cannot_reenter_its_cursor(cursor, tvp_schema, operation): + def rows(): + yield 1, "first" + if operation == "execute": + cursor.execute("SELECT 99") + else: + cursor.close() + yield 2, "second" + + with pytest.raises(DatabaseError, match="while it is executing"): + cursor.execute(f"EXEC [{tvp_schema}].CountRows ?", items(tvp_schema, rows())) + assert cursor.execute("SELECT 42").fetchone()[0] == 42 + + +def _run_isolated_stream_checks(mode, schema): + connection_string = os.environ["DB_CONNECTION_STRING"] + if mode == "forwarder": + target = _parse_server(connection_string) + assert target is not None + connection_string = _replace_server(connection_string, *_start_forwarder(target)) + connection = connect(connection_string, autocommit=True) + cursor = connection.cursor() + try: + if mode == "disconnect": + original = ValueError("original source failure after disconnect") + source_errors = [] - slot.value = ctypes.cast(invalid_metadata_column, ctypes.c_void_p).value + def rows(): + yield 1, "first" try: - with pytest.raises(ProgrammingError, match="(?i)invalid descriptor index"): - cursor.execute(sql, ([(1,) + padding],)) - finally: - slot.value = original - cursor.execute("SELECT ?", 42) - assert [tuple(row) for row in cursor.fetchall()] == [(42,)] - - # No pointer overrides on the normal path, including after error recovery. - for ids in ([1, 2], [3], []): - cursor.execute(sql, ([(i,) + padding for i in ids],)) - assert [tuple(row) for row in cursor.fetchall()] == [(i,) for i in ids] - print("OK forwarded TVP metadata", flush=True) + connection.close() + except DatabaseError as error: + source_errors.append(error) + raise + source_errors.append(original) + raise original + + try: + cursor.execute(f"EXEC [{schema}].CountRows ?", items(schema, rows())) + except (ValueError, DatabaseError) as error: + assert error is source_errors[0] + else: + raise AssertionError("Lost the original iterator error") + print("OK TVP streaming") + return + elif mode == "cancel": + waiting = threading.Event() + resume = threading.Event() + errors = [] + + def rows(): + yield 1, "first" + waiting.set() + assert resume.wait(10) + yield 2, "second" + + def execute(): + try: + cursor.execute(f"EXEC [{schema}].CountRows ?", items(schema, rows())) + except BaseException as error: + errors.append(error) + + worker = threading.Thread(target=execute) + worker.start() + assert waiting.wait(10) + try: + cursor.hstmt._cancel() + finally: + resume.set() + worker.join(10) + assert not worker.is_alive() + assert len(errors) == 1 and isinstance(errors[0], DatabaseError) + else: + count = 20000 if mode == "memory" else 1000 + if mode == "memory": + import resource + + before = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss + cursor.execute( + f"EXEC [{schema}].CountRows ?", + items(schema, ((i, str(i).ljust(4096, "x")) for i in range(count))), + ) + assert tuple(cursor.fetchone()) == (count, count * (count - 1) // 2) + if mode == "memory": + growth = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss - before + if sys.platform != "darwin": + growth *= 1024 + # Input exceeds 80MB of Python text / 160MB UTF-16. Additional + # native and Python memory must stay below a table-sized copy. + assert growth < 64 * 1024 * 1024, growth + print(f"peak RSS growth: {growth} bytes") + assert cursor.execute("SELECT 42").fetchone()[0] == 42 + finally: + cursor.close() + connection.close() + print("OK TVP streaming") @pytest.mark.parametrize( - "metadata_error", + "mode", [ - pytest.param(False, id="metadata-reads"), + "cancel", + "forwarder", + "disconnect", pytest.param( - True, - id="metadata-error-cleanup", - marks=pytest.mark.skipif( - sys.platform == "win32", reason="Native function-pointer injection is Unix-only" - ), + "memory", + marks=pytest.mark.skipif(sys.platform == "win32", reason="POSIX resource accounting"), ), ], ) -def test_tvp_metadata_through_python_forwarder_does_not_deadlock( - conn_str, wide_tvp_procedure, metadata_error -): - if not conn_str or _parse_server(conn_str) is None: - pytest.skip("Requires DB_CONNECTION_STRING with Server=host,port") +def test_isolated_stream_lifecycle(conn_str, tvp_schema, mode): + if mode == "forwarder" and _parse_server(conn_str) is None: + pytest.skip("Requires a Server=host,port connection string") env = os.environ.copy() env["PYTHONPATH"] = os.pathsep.join(sys.path) - env["TVP_GIL_PROCEDURE"] = wide_tvp_procedure - env["TVP_GIL_METADATA_ERROR"] = str(int(metadata_error)) - try: - result = subprocess.run( - [ - sys.executable, - "-c", - "from test_028_table_valued_parameters import _run_forwarded_tvp; " - "_run_forwarded_tvp()", - ], - env=env, - capture_output=True, - text=True, - timeout=WATCHDOG_SECONDS, - ) - except subprocess.TimeoutExpired: - pytest.fail( - f"TVP metadata through the Python forwarder deadlocked after {WATCHDOG_SECONDS}s" - ) + result = subprocess.run( + [ + sys.executable, + "-c", + "from test_028_table_valued_parameters import _run_isolated_stream_checks;" + f"_run_isolated_stream_checks({mode!r}, {tvp_schema!r})", + ], + env=env, + capture_output=True, + text=True, + timeout=WATCHDOG_SECONDS, + ) assert result.returncode == 0, result.stdout + result.stderr - assert "OK forwarded TVP metadata" in result.stdout + assert "OK TVP streaming" in result.stdout + print(result.stdout, end="") From ce598d4f3a20096ccad6e9f6710394df96b120e8 Mon Sep 17 00:00:00 2001 From: Gaurav Sharma Date: Tue, 13 Oct 2026 13:54:51 +0530 Subject: [PATCH 08/15] FIX: Restore TVP CI compatibility on GCC and LocalDB Give the TVP state external linkage with hidden GNU visibility so the public ParamInfo binding does not contain an anonymous-namespace member type. Keep the existing compiler warning policy intact. Limit the TCP-forwarder regression to TCP endpoints instead of routing LocalDB and named-pipe instances through a socket proxy. Other lifecycle and LocalDB cases remain enabled. AB#48177 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- mssql_python/pybind/ddbc_bindings.cpp | 4 ++++ mssql_python/pybind/param_detect.hpp | 8 +++++++- tests/test_028_table_valued_parameters.py | 6 ++++-- 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/mssql_python/pybind/ddbc_bindings.cpp b/mssql_python/pybind/ddbc_bindings.cpp index 0e7630c21..1a12d1ac7 100644 --- a/mssql_python/pybind/ddbc_bindings.cpp +++ b/mssql_python/pybind/ddbc_bindings.cpp @@ -894,8 +894,12 @@ static void PrepareParamBuffer(const py::handle& param, ParamInfo& paramInfo, } } +} // namespace + #include "tvp.hpp" +namespace { + // Given a list of parameters and their ParamInfo, calls SQLBindParameter on // each of them with appropriate arguments SQLRETURN BindParameters(const SqlHandlePtr& statementHandle, SQLHANDLE hStmt, const py::list& params, diff --git a/mssql_python/pybind/param_detect.hpp b/mssql_python/pybind/param_detect.hpp index 61f2c0a68..19c03bbde 100644 --- a/mssql_python/pybind/param_detect.hpp +++ b/mssql_python/pybind/param_detect.hpp @@ -38,6 +38,13 @@ // Parameter description types //------------------------------------------------------------------------------------------------- +// External linkage for ParamInfo ownership; match its pybind11 members' visibility. +#ifdef __GNUC__ +struct __attribute__((visibility("hidden"))) TableParamState; +#else +struct TableParamState; +#endif + // This struct is shared between C++ & Python code. // Suppress -Wattributes warning for ParamInfo struct // The warning is triggered because pybind11 handles visibility attributes automatically, @@ -46,7 +53,6 @@ #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wattributes" #endif -namespace { struct TableParamState; } struct ParamInfo { SQLSMALLINT inputOutputType = SQL_PARAM_INPUT; SQLSMALLINT paramCType = SQL_C_DEFAULT; diff --git a/tests/test_028_table_valued_parameters.py b/tests/test_028_table_valued_parameters.py index 598ebb542..38b13344c 100644 --- a/tests/test_028_table_valued_parameters.py +++ b/tests/test_028_table_valued_parameters.py @@ -649,8 +649,10 @@ def execute(): ], ) def test_isolated_stream_lifecycle(conn_str, tvp_schema, mode): - if mode == "forwarder" and _parse_server(conn_str) is None: - pytest.skip("Requires a Server=host,port connection string") + if mode == "forwarder": + target = _parse_server(conn_str) + if target is None or "\\" in target[0]: + pytest.skip("TCP forwarding requires a host, not a LocalDB or named-pipe instance") env = os.environ.copy() env["PYTHONPATH"] = os.pathsep.join(sys.path) result = subprocess.run( From a788a374f74f49752020eab8f50ba9d09afc1d41 Mon Sep 17 00:00:00 2001 From: Gaurav Sharma Date: Tue, 13 Oct 2026 16:04:50 +0530 Subject: [PATCH 09/15] CHORE: Move TVP memory coverage to stress CI Keep the large TVP memory workload out of per-PR validation and run it in the existing Linux stress step. Preserve the row count and RSS bound, allow slower stress transfers, and include child stack traces in timeout failures. Retry dependency installation tasks twice across Windows, Unix, and container jobs without ignoring persistent failures. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- OneBranchPipelines/stress-test-pipeline.yml | 3 +- eng/pipelines/steps/install-mssql-py-core.yml | 3 ++ tests/test_028_table_valued_parameters.py | 42 +++++++++++++------ 3 files changed, 34 insertions(+), 14 deletions(-) diff --git a/OneBranchPipelines/stress-test-pipeline.yml b/OneBranchPipelines/stress-test-pipeline.yml index 548f52186..4cd98d9fe 100644 --- a/OneBranchPipelines/stress-test-pipeline.yml +++ b/OneBranchPipelines/stress-test-pipeline.yml @@ -387,7 +387,7 @@ jobs: # Run original stress tests - script: | - python -m pytest tests/test_011_singlethreaded_stress.py -v -m "stress" --junitxml=perf-stress-test-results-linux.xml --timeout=1800 --capture=tee-sys + python -m pytest tests/test_011_singlethreaded_stress.py "tests/test_028_table_valued_parameters.py::test_isolated_stream_lifecycle[memory]" -v -m "stress" --junitxml=perf-stress-test-results-linux.xml --timeout=1800 --capture=tee-sys displayName: 'Run Single-Threaded Stress Tests' env: DB_CONNECTION_STRING: 'Server=localhost;Database=StressTestDB;Uid=testuser;Pwd=$(DB_PASSWORD);TrustServerCertificate=yes' @@ -437,6 +437,7 @@ jobs: echo "Tests executed:" echo " - test_020_multithreaded_stress.py (stress marker)" echo " - test_011_singlethreaded_stress.py (stress marker)" + echo " - test_028_table_valued_parameters.py::test_isolated_stream_lifecycle[memory] (Linux stress)" echo "" echo "See individual job results for details." displayName: 'Print Summary' diff --git a/eng/pipelines/steps/install-mssql-py-core.yml b/eng/pipelines/steps/install-mssql-py-core.yml index a10cf0b46..f0e012f4a 100644 --- a/eng/pipelines/steps/install-mssql-py-core.yml +++ b/eng/pipelines/steps/install-mssql-py-core.yml @@ -40,6 +40,7 @@ steps: - ${{ if eq(parameters.platform, 'windows') }}: - task: PowerShell@2 displayName: 'Install mssql-python-rs${{ parameters.displaySuffix }}' + retryCountOnTaskFailure: 2 inputs: targetType: 'filePath' filePath: 'eng/scripts/install-mssql-py-core.ps1' @@ -50,6 +51,7 @@ steps: chmod +x eng/scripts/install-mssql-py-core.sh ./eng/scripts/install-mssql-py-core.sh displayName: 'Install mssql-python-rs${{ parameters.displaySuffix }}' + retryCountOnTaskFailure: 2 # Inside a Docker container - ${{ if eq(parameters.platform, 'container') }}: @@ -60,3 +62,4 @@ steps: ./eng/scripts/install-mssql-py-core.sh " displayName: 'Install mssql-python-rs in ${{ parameters.containerName }}${{ parameters.displaySuffix }}' + retryCountOnTaskFailure: 2 diff --git a/tests/test_028_table_valued_parameters.py b/tests/test_028_table_valued_parameters.py index 38b13344c..3c7da930e 100644 --- a/tests/test_028_table_valued_parameters.py +++ b/tests/test_028_table_valued_parameters.py @@ -644,7 +644,10 @@ def execute(): "disconnect", pytest.param( "memory", - marks=pytest.mark.skipif(sys.platform == "win32", reason="POSIX resource accounting"), + marks=[ + pytest.mark.stress, + pytest.mark.skipif(sys.platform == "win32", reason="POSIX resource accounting"), + ], ), ], ) @@ -655,18 +658,31 @@ def test_isolated_stream_lifecycle(conn_str, tvp_schema, mode): pytest.skip("TCP forwarding requires a host, not a LocalDB or named-pipe instance") env = os.environ.copy() env["PYTHONPATH"] = os.pathsep.join(sys.path) - result = subprocess.run( - [ - sys.executable, - "-c", - "from test_028_table_valued_parameters import _run_isolated_stream_checks;" - f"_run_isolated_stream_checks({mode!r}, {tvp_schema!r})", - ], - env=env, - capture_output=True, - text=True, - timeout=WATCHDOG_SECONDS, - ) + # The memory stress case sends 160MB, unlike the short deadlock probes. + # Keep the same row count and RSS bound, but allow slower transport. + timeout = 120 if mode == "memory" else WATCHDOG_SECONDS + try: + result = subprocess.run( + [ + sys.executable, + "-u", + "-c", + "import faulthandler;" + f"faulthandler.dump_traceback_later({timeout - 5});" + "from test_028_table_valued_parameters import _run_isolated_stream_checks;" + f"_run_isolated_stream_checks({mode!r}, {tvp_schema!r})", + ], + env=env, + capture_output=True, + text=True, + timeout=timeout, + ) + except subprocess.TimeoutExpired as error: + output = "".join( + chunk.decode(errors="replace") if isinstance(chunk, bytes) else chunk or "" + for chunk in (error.stdout, error.stderr) + ) + pytest.fail(f"TVP {mode} subprocess exceeded {timeout}s:\n{output}") assert result.returncode == 0, result.stdout + result.stderr assert "OK TVP streaming" in result.stdout print(result.stdout, end="") From 0d950144bccd9ea75bdf8d7dd0f0bd904907e586 Mon Sep 17 00:00:00 2001 From: Gaurav Sharma Date: Tue, 13 Oct 2026 17:18:58 +0530 Subject: [PATCH 10/15] FIX: Guard optional cancellation during parameter cleanup Preserve the original execution error when the driver does not export SQLCancel, matching the existing optional-cancellation contract. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- mssql_python/pybind/ddbc_bindings.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mssql_python/pybind/ddbc_bindings.cpp b/mssql_python/pybind/ddbc_bindings.cpp index 1a12d1ac7..03dcf889c 100644 --- a/mssql_python/pybind/ddbc_bindings.cpp +++ b/mssql_python/pybind/ddbc_bindings.cpp @@ -2334,7 +2334,7 @@ SQLRETURN SQLExecute_wrap(const SqlHandlePtr statementHandle, SQLRETURN resetResult; { py::gil_scoped_release release; - SQLCancel_ptr(hStmt); + if (SQLCancel_ptr) SQLCancel_ptr(hStmt); resetResult = SQLFreeStmt_ptr(hStmt, SQL_RESET_PARAMS); } SQLRETURN freeResult = statementHandle->freeHandle(); From 8532a0123aeb4b9e3b77b5286b75db6a6ba4baa1 Mon Sep 17 00:00:00 2001 From: Gaurav Sharma Date: Tue, 13 Oct 2026 17:20:26 +0530 Subject: [PATCH 11/15] FIX: Release Python owners from failed parameter storage Drop scalar and TVP Python references while holding the GIL after an execution failure. Keep bound native tokens, indicators, metadata, and buffers alive without letting retained storage decref Python objects after interpreter finalization. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- mssql_python/pybind/ddbc_bindings.cpp | 10 ++++++++-- mssql_python/pybind/tvp.hpp | 7 +++++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/mssql_python/pybind/ddbc_bindings.cpp b/mssql_python/pybind/ddbc_bindings.cpp index 03dcf889c..75ae3d2ca 100644 --- a/mssql_python/pybind/ddbc_bindings.cpp +++ b/mssql_python/pybind/ddbc_bindings.cpp @@ -2339,9 +2339,15 @@ SQLRETURN SQLExecute_wrap(const SqlHandlePtr statementHandle, } SQLRETURN freeResult = statementHandle->freeHandle(); if (!SQL_SUCCEEDED(resetResult) && !SQL_SUCCEEDED(freeResult)) { - paramBuffers.push_back( - std::make_shared>(std::move(paramInfos))); + auto retainedInfos = std::make_shared>(std::move(paramInfos)); + paramBuffers.push_back(retainedInfos); statementHandle->retainFailedParameters(std::move(paramBuffers)); + // Bound tokens/indicators are native storage. Drop Python owners + // with the GIL held, before retained storage can outlive Python. + for (auto& info : *retainedInfos) { + info.dataPtr = py::object(); + if (info.table) info.table->releasePythonState(); + } } } throw; diff --git a/mssql_python/pybind/tvp.hpp b/mssql_python/pybind/tvp.hpp index f7a762cd5..eb07dd737 100644 --- a/mssql_python/pybind/tvp.hpp +++ b/mssql_python/pybind/tvp.hpp @@ -86,6 +86,13 @@ struct TableParamState { } return true; } + + void releasePythonState() { + value = py::object(); + iterator = py::object(); + sizes.release().dec_ref(); + row.release().dec_ref(); + } }; static SQLRETURN BindTableValuedParameter(const SqlHandlePtr& handle, int paramIndex, From 1dd4c430786c5ac380ef24359a9a23b19d551eac Mon Sep 17 00:00:00 2001 From: Gaurav Sharma Date: Tue, 13 Oct 2026 17:27:12 +0530 Subject: [PATCH 12/15] FIX: Retain failed parameter buffers through native teardown Keep failed statement storage owned by the connection after its Python-facing handle wrapper is dropped. Release storage after successful statement free or parent disconnect, and leave still-bound native storage to process cleanup if even parent teardown fails. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- mssql_python/pybind/connection/connection.cpp | 4 +++ mssql_python/pybind/ddbc_bindings.cpp | 33 ++++++++++++++++--- mssql_python/pybind/ddbc_bindings.h | 19 ++++++++--- 3 files changed, 46 insertions(+), 10 deletions(-) diff --git a/mssql_python/pybind/connection/connection.cpp b/mssql_python/pybind/connection/connection.cpp index 9c4304d2f..8bbff9066 100644 --- a/mssql_python/pybind/connection/connection.cpp +++ b/mssql_python/pybind/connection/connection.cpp @@ -173,6 +173,10 @@ void Connection::disconnect(bool rollbackBeforeDisconnect) { for (const auto& handle : childHandles) { handle->markImplicitlyFreed(); } + for (const auto& buffers : _cleanupState->failedParameterBuffers) { + buffers->release(); + } + _cleanupState->failedParameterBuffers.clear(); _childStatementHandles.clear(); _allocationsSinceCompaction = 0; } diff --git a/mssql_python/pybind/ddbc_bindings.cpp b/mssql_python/pybind/ddbc_bindings.cpp index 75ae3d2ca..2f27590a5 100644 --- a/mssql_python/pybind/ddbc_bindings.cpp +++ b/mssql_python/pybind/ddbc_bindings.cpp @@ -1621,6 +1621,18 @@ void DriverLoader::loadDriver() { } } +FailedParameterBuffers::FailedParameterBuffers(std::vector> buffers) + : _buffers(std::make_unique>>(std::move(buffers))) {} + +FailedParameterBuffers::~FailedParameterBuffers() { + if (_buffers) { + // If even parent teardown failed, the driver may still use these addresses. + // Abandon native storage to process teardown rather than free bound buffers. + std::fputs("mssql-python: retaining parameter storage after failed native teardown\n", stderr); + _buffers.release(); + } +} + // SqlHandle definition SqlHandle::SqlHandle(SQLSMALLINT type, SQLHANDLE rawHandle, std::shared_ptr cleanupState) @@ -1640,7 +1652,18 @@ std::unique_lock SqlHandle::lockForCleanup() const { } SQLHANDLE SqlHandle::get() const { - return _failedParameterBuffers.empty() ? _handle : nullptr; + return _failedParameterBuffers ? nullptr : _handle; +} + +void SqlHandle::retainFailedParameters(std::vector> buffers) { + _failedParameterBuffers = std::make_shared(std::move(buffers)); + py::gil_scoped_release release; + auto cleanupLock = lockForCleanup(); + if (!_handle || _implicitly_freed || (_cleanupState && _cleanupState->disconnected)) { + _failedParameterBuffers->release(); + } else if (_cleanupState) { + _cleanupState->failedParameterBuffers.push_back(_failedParameterBuffers); + } } SQLSMALLINT SqlHandle::type() const { @@ -1698,13 +1721,13 @@ SQLRETURN SqlHandle::freeHandle() { return SQL_INVALID_HANDLE; } clearDescribeCache(); - if (_implicitly_freed || (_cleanupState && _cleanupState->disconnected)) { - _handle = nullptr; - return SQL_SUCCESS; + SQLRETURN ret = SQL_SUCCESS; + if (!_implicitly_freed && !(_cleanupState && _cleanupState->disconnected)) { + ret = SQLFreeHandle_ptr(_type, _handle); } - SQLRETURN ret = SQLFreeHandle_ptr(_type, _handle); if (SQL_SUCCEEDED(ret)) { _handle = nullptr; + if (_failedParameterBuffers) _failedParameterBuffers->release(); } return ret; }; diff --git a/mssql_python/pybind/ddbc_bindings.h b/mssql_python/pybind/ddbc_bindings.h index e0d47ad40..471fcab87 100644 --- a/mssql_python/pybind/ddbc_bindings.h +++ b/mssql_python/pybind/ddbc_bindings.h @@ -292,9 +292,20 @@ struct DescribedParamInfo { SQLSMALLINT decimalDigits; }; +class FailedParameterBuffers { + public: + explicit FailedParameterBuffers(std::vector> buffers); + ~FailedParameterBuffers(); + void release() { _buffers.reset(); } + + private: + std::unique_ptr>> _buffers; +}; + struct ConnectionCleanupState { std::mutex mutex; bool disconnected = false; // Protected by mutex, shared with every child. + std::vector> failedParameterBuffers; }; class SqlHandle { @@ -315,9 +326,7 @@ class SqlHandle { // unavailable. void cancel(); bool isImplicitlyFreed() const { return _implicitly_freed; } - void retainFailedParameters(std::vector> buffers) { - _failedParameterBuffers = std::move(buffers); - } + void retainFailedParameters(std::vector> buffers); // Mark this handle as implicitly freed (freed by parent handle) // This prevents double-free attempts when the ODBC driver automatically @@ -350,8 +359,8 @@ class SqlHandle { bool _implicitly_freed = false; // Tracks if handle was freed by parent std::shared_ptr _cleanupState; // Only populated if both parameter reset and handle release fail. The - // handle is then unusable, but its bound storage lives through teardown. - std::vector> _failedParameterBuffers; + // The connection also retains this native storage if the wrapper is dropped. + std::shared_ptr _failedParameterBuffers; }; using SqlHandlePtr = std::shared_ptr; From 0c344d349de75566983d60585e6f76e846c9e43c Mon Sep 17 00:00:00 2001 From: Gaurav Sharma Date: Tue, 13 Oct 2026 18:26:23 +0530 Subject: [PATCH 13/15] CHORE: Keep TVP support scoped to synchronous execution Retain the shared explicit TVP value and synchronous ODBC streaming. Remove async adaptation, async-only input types, tests and documentation so async integration can be owned separately. Remove unrelated dependency-install retries while preserving native cleanup fixes and the existing Linux memory-stress coverage. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- CHANGELOG.md | 4 +- README.md | 14 +-- eng/pipelines/steps/install-mssql-py-core.yml | 3 - mssql_python/async_query/async_execute.py | 40 +------ mssql_python/cursor.py | 11 -- mssql_python/mssql_python.pyi | 3 - mssql_python/table_valued_parameter.py | 4 +- tests/test_028_table_valued_parameters.py | 113 +++--------------- 8 files changed, 24 insertions(+), 168 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 27b946cfa..ae2e76dbf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,9 +10,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - **AB#48177:** Added explicit `TableValuedParameter(type_name, columns, rows, schema=...)` input values. Sync `execute()` streams lists, tuples and iterators through native ODBC data-at-execution using one row of additional storage and - one procedure invocation, without discovering table metadata. Async execution - adapts materialized rows to the pinned native core; iterator streaming is not - available asynchronously. See README for current native async schema limits. + one procedure invocation, without discovering table metadata. - New feature: Support for macOS and Linux. - Documentation: Added API documentation in the Wiki. - New `token_provider=` parameter on `connect()` / `Connection` for Microsoft diff --git a/README.md b/README.md index 9a879beee..e5553910b 100644 --- a/README.md +++ b/README.md @@ -219,7 +219,7 @@ CREATE PROCEDURE dbo.ProcessItems @items dbo.ItemBatch READONLY AS SELECT id, label FROM @items; ``` -Pass one explicit value to `execute()`; ordinary nested lists/tuples are **not** +Pass one explicit value to synchronous `execute()`; ordinary nested lists/tuples are **not** implicitly treated as TVPs: ```python @@ -238,7 +238,7 @@ driver's integer `SQL_*` constants. Columns bind **positionally**; names are diagnostic labels, not a schema lookup. Character and binary columns require a size; `0` means MAX for variable-length types. Decimal columns require both precision and scale, e.g. `("amount", SQL_DECIMAL, 20, 6)`. Precision is 1–38; -scale is 0–precision. Temporal scale defaults to 7 (0–7 supported synchronously). +scale is 0–precision. Temporal scale defaults to 7 (0–7 supported). Sizes for Unicode columns are UTF-16 code units; binary sizes are bytes. Rows must contain one cell per column. Supported cells include integers, floats, bools, `Decimal`, strings, bytes/bytearray, date/time/datetime, UUID and `None`. @@ -260,16 +260,6 @@ before releasing its buffers. `executemany()` and `setinputsizes()` overrides for statements containing TVPs are not supported. The `mssql-odbc` provider does not currently support this ODBC TVP path. -The **same Python value** is accepted by preview async `execute()`, with -materialized list/tuple rows only. The pinned `mssql-python-rs` 0.3.0 constructor -materializes its native table; generators and async generators raise -`NotSupportedError` **without being consumed**, not a hidden `list()` conversion. -Its current TVP metadata builder additionally requires **DECIMAL/NUMERIC(18,10)** -and **temporal scale 7**, even when every cell is non-NULL. Other declarations -propagate the native error. Native streaming and arbitrary decimal/temporal -metadata require follow-up in `mssql-rs`; this driver does not implement or -promise those Rust features. - ## Still have questions? Check out our [FAQ](https://github.com/microsoft/mssql-python/wiki/Frequently-Asked-Questions). Still not answered? Create an [issue](https://github.com/microsoft/mssql-python/issues/new/choose) to ask a question. diff --git a/eng/pipelines/steps/install-mssql-py-core.yml b/eng/pipelines/steps/install-mssql-py-core.yml index f0e012f4a..a10cf0b46 100644 --- a/eng/pipelines/steps/install-mssql-py-core.yml +++ b/eng/pipelines/steps/install-mssql-py-core.yml @@ -40,7 +40,6 @@ steps: - ${{ if eq(parameters.platform, 'windows') }}: - task: PowerShell@2 displayName: 'Install mssql-python-rs${{ parameters.displaySuffix }}' - retryCountOnTaskFailure: 2 inputs: targetType: 'filePath' filePath: 'eng/scripts/install-mssql-py-core.ps1' @@ -51,7 +50,6 @@ steps: chmod +x eng/scripts/install-mssql-py-core.sh ./eng/scripts/install-mssql-py-core.sh displayName: 'Install mssql-python-rs${{ parameters.displaySuffix }}' - retryCountOnTaskFailure: 2 # Inside a Docker container - ${{ if eq(parameters.platform, 'container') }}: @@ -62,4 +60,3 @@ steps: ./eng/scripts/install-mssql-py-core.sh " displayName: 'Install mssql-python-rs in ${{ parameters.containerName }}${{ parameters.displaySuffix }}' - retryCountOnTaskFailure: 2 diff --git a/mssql_python/async_query/async_execute.py b/mssql_python/async_query/async_execute.py index 8c5f8b4a9..6f9bb7b05 100644 --- a/mssql_python/async_query/async_execute.py +++ b/mssql_python/async_query/async_execute.py @@ -4,9 +4,7 @@ from collections.abc import Mapping, Sequence from typing import TYPE_CHECKING, Any -from ..exceptions import NotSupportedError, OperationalError, ProgrammingError -from ..table_valued_parameter import TableValuedParameter -from ._native import load_py_core +from ..exceptions import OperationalError, ProgrammingError from ..logging import logger from ..row import Row from .exception_translator import translate_py_core_exceptions @@ -52,42 +50,6 @@ async def execute( if len(parameters) == 1 and isinstance(parameters[0], (tuple, list, Row)): parameters = tuple(parameters[0]) - def adapt(value): - if not isinstance(value, TableValuedParameter): - return value - if not isinstance(value.rows, (list, tuple)): - raise NotSupportedError( - "Async TVPs currently require materialized list/tuple rows", - "Use synchronous execute for streaming; native async TVP streaming is not available", - ) - for index, row in enumerate(value.rows): - value._validate_row(row, index) - return load_py_core().TableValuedParameter( - value.type_name, - # py-core selects MAX above the inline width, whereas ODBC uses 0. - columns=[ - ( - {-1: 12, -10: -9, -4: -3}.get(sql_type, sql_type), - 2147483647 if size == 0 and sql_type in (12, -9, -3, -1, -10, -4) else size, - scale, - ) - for _, sql_type, size, scale in value.columns - ], - rows=value.rows, - schema=value.schema, - ) - - # Named parameter dictionaries are passed intact to py-core, just like - # ordinary execute; adapt only explicit TVP values. - parameters = tuple( - ( - {key: adapt(value) for key, value in parameter.items()} - if isinstance(parameter, Mapping) - else adapt(parameter) - ) - for parameter in parameters - ) - logger.debug( "AsyncCursor.execute: starting; param_count=%d; use_prepare=%s; reset_cursor=%s", len(parameters), diff --git a/mssql_python/cursor.py b/mssql_python/cursor.py index abefae2c3..72a81a518 100644 --- a/mssql_python/cursor.py +++ b/mssql_python/cursor.py @@ -16,7 +16,6 @@ import uuid import datetime import warnings -from collections.abc import AsyncIterable from typing import List, Mapping, Union, Any, Optional, Tuple, Sequence, TYPE_CHECKING, Iterable from mssql_python.constants import ConstantsDDBC as ddbc_sql_const, SQLTypes from mssql_python.helpers import check_error, connstr_to_pycore_params @@ -1844,16 +1843,6 @@ def execute( # pylint: disable=too-many-locals,too-many-branches,too-many-state # Getting encoding setting encoding_settings = self._get_encoding_settings() - if any( - isinstance(parameter, TableValuedParameter) - and isinstance(parameter.rows, AsyncIterable) - for parameter in parameters - ): - raise NotSupportedError( - "Synchronous TVPs require a synchronous row source", - "Use a list, tuple, or synchronous iterator", - ) - # Validate that inputsizes matches parameter count if both are present if parameters and self._inputsizes: if any(isinstance(parameter, TableValuedParameter) for parameter in parameters): diff --git a/mssql_python/mssql_python.pyi b/mssql_python/mssql_python.pyi index 3215ce7b5..8e0df1951 100644 --- a/mssql_python/mssql_python.pyi +++ b/mssql_python/mssql_python.pyi @@ -16,7 +16,6 @@ from typing import ( Callable, Iterator, Iterable, - AsyncIterable, ) import datetime import logging @@ -31,7 +30,6 @@ class TableValuedParameter: List[Sequence[Any]], Tuple[Sequence[Any], ...], Iterator[Sequence[Any]], - AsyncIterable[Sequence[Any]], ], *, schema: Optional[str] = None, @@ -49,7 +47,6 @@ class TableValuedParameter: List[Sequence[Any]], Tuple[Sequence[Any], ...], Iterator[Sequence[Any]], - AsyncIterable[Sequence[Any]], ]: ... # GLOBALS - DB-API 2.0 Required Module Globals diff --git a/mssql_python/table_valued_parameter.py b/mssql_python/table_valued_parameter.py index 770324521..69391a986 100644 --- a/mssql_python/table_valued_parameter.py +++ b/mssql_python/table_valued_parameter.py @@ -1,6 +1,6 @@ """Explicit, transport-neutral input values for SQL Server table types.""" -from collections.abc import AsyncIterable, Iterator +from collections.abc import Iterator from datetime import date, datetime, time from decimal import Decimal from uuid import UUID @@ -78,7 +78,7 @@ def __init__(self, type_name, columns, rows, *, schema=None): ): raise ValueError(f"TVP column {name}: fixed-length columns require a positive size") normalized.append((name, sql_type, size, scale)) - if not isinstance(rows, (list, tuple, Iterator, AsyncIterable)): + if not isinstance(rows, (list, tuple, Iterator)): raise TypeError("TVP rows must be a list, tuple, or iterator; use [] for an empty TVP") self._type_name, self._schema = type_name, schema self._columns, self._rows = tuple(normalized), rows diff --git a/tests/test_028_table_valued_parameters.py b/tests/test_028_table_valued_parameters.py index 3c7da930e..731116c05 100644 --- a/tests/test_028_table_valued_parameters.py +++ b/tests/test_028_table_valued_parameters.py @@ -1,8 +1,6 @@ -"""Live SQL Server coverage for the shared explicit TVP contract and ODBC streaming.""" +"""Live SQL Server coverage for explicit TVPs and synchronous ODBC streaming.""" -import asyncio import datetime -import inspect import os import subprocess import sys @@ -12,7 +10,6 @@ from decimal import Decimal, localcontext import pytest -import pytest_asyncio from mssql_python import ( DatabaseError, @@ -33,7 +30,6 @@ SQL_TYPE_TIMESTAMP, SQL_GUID, ) -from mssql_python.async_query import _AsyncConnection from mssql_python.constants import ConstantsDDBC from test_023_ssh_tunnel_gil_release import ( WATCHDOG_SECONDS, @@ -94,29 +90,9 @@ def items(schema, rows): ) -async def call(method, *args, **kwargs): - result = method(*args, **kwargs) - return await result if inspect.isawaitable(result) else result - - -@pytest_asyncio.fixture(params=["sync", "async"]) -async def tvp_cursor(request, conn_str): - if request.param == "sync": - connection = connect(conn_str, autocommit=True) - else: - connection = await _AsyncConnection.connect(conn_str, autocommit=True) - cursor = connection.cursor() - try: - yield cursor - finally: - await call(cursor.close) - await call(connection.close) - - -@pytest.mark.asyncio @pytest.mark.parametrize("rows", [[], [(1, "Keyboard"), (2, None)], ((3, ""),)]) @pytest.mark.parametrize("named", [False, True]) -async def test_common_rows_named_reuse_and_one_invocation(tvp_cursor, tvp_schema, rows, named): +def test_rows_named_reuse_and_one_invocation(cursor, tvp_schema, rows, named): value = items(tvp_schema, rows) sql = f"EXEC [{tvp_schema}].Echo ?, ?, ?" params = (10, value, "✓") @@ -124,38 +100,34 @@ async def test_common_rows_named_reuse_and_one_invocation(tvp_cursor, tvp_schema sql = f"EXEC [{tvp_schema}].Echo %(prefix)s, %(items)s, %(suffix)s" params = {"prefix": 10, "items": value, "suffix": "✓"} for _ in range(2): - await call(tvp_cursor.execute, sql, params) - result = await call(tvp_cursor.fetchall) + cursor.execute(sql, params) + result = cursor.fetchall() assert [tuple(row) for row in result] == [ (row[0] + 10, None if row[1] is None else row[1] + "✓") for row in rows ] - await call(tvp_cursor.execute, f"SELECT COUNT(*) FROM [{tvp_schema}].Calls") - assert (await call(tvp_cursor.fetchone))[0] == 2 + cursor.execute(f"SELECT COUNT(*) FROM [{tvp_schema}].Calls") + assert cursor.fetchone()[0] == 2 -@pytest.mark.asyncio @pytest.mark.parametrize("left,right", [([], []), ([], [(2, "right")]), ([(1, None)], [(2, "")])]) -async def test_common_multiple_tvps(tvp_cursor, tvp_schema, left, right): - await call( - tvp_cursor.execute, +def test_multiple_tvps(cursor, tvp_schema, left, right): + cursor.execute( f"EXEC [{tvp_schema}].Pair ?, ?", items(tvp_schema, left), items(tvp_schema, right), ) - result = await call(tvp_cursor.fetchall) + result = cursor.fetchall() assert sorted(tuple(row) for row in result) == sorted(left + right) -@pytest.mark.asyncio -async def test_common_null_scalars_with_tvp(tvp_cursor, tvp_schema): - await call( - tvp_cursor.execute, +def test_null_scalars_with_tvp(cursor, tvp_schema): + cursor.execute( f"EXEC [{tvp_schema}].Echo ?, ?, ?", None, items(tvp_schema, [(1, "one")]), None, ) - assert tuple(await call(tvp_cursor.fetchone)) == (None, None) + assert tuple(cursor.fetchone()) == (None, None) def test_explicit_type_in_parameterized_batch(cursor, tvp_schema): @@ -188,8 +160,7 @@ def test_declared_text_size_failure_recovers(cursor, tvp_schema): assert cursor.execute("SELECT 42").fetchone()[0] == 42 -@pytest.mark.asyncio -async def test_common_typed_cells(tvp_cursor, cursor, db_connection): +def test_typed_cells(cursor, db_connection): suffix = uuid.uuid4().hex name, proc = f"Typed_{suffix}", f"TypedProc_{suffix}" declarations = [ @@ -255,8 +226,8 @@ async def test_common_typed_cells(tvp_cursor, cursor, db_connection): ctx.prec = 5 source = [list(row), [None] * len(row)] value = TableValuedParameter(f"dbo.{name}", columns, source) - await call(tvp_cursor.execute, f"EXEC dbo.[{proc}] ?", value) - results = [tuple(r) for r in await call(tvp_cursor.fetchall)] + cursor.execute(f"EXEC dbo.[{proc}] ?", value) + results = [tuple(r) for r in cursor.fetchall()] assert len(results) == 2 actual = next(r for r in results if r[0] is not None) expected = list(row) @@ -272,8 +243,7 @@ async def test_common_typed_cells(tvp_cursor, cursor, db_connection): db_connection.commit() -@pytest.mark.asyncio -async def test_common_date_range(tvp_cursor, cursor, db_connection): +def test_date_range(cursor, db_connection): name = f"Dates_{uuid.uuid4().hex}" rows = [ (None,), @@ -286,8 +256,8 @@ async def test_common_date_range(tvp_cursor, cursor, db_connection): db_connection.commit() try: value = TableValuedParameter(f"dbo.{name}", [("value", SQL_TYPE_DATE)], rows) - await call(tvp_cursor.execute, "SELECT value FROM ? ORDER BY value", value) - assert [tuple(row) for row in await call(tvp_cursor.fetchall)] == rows + cursor.execute("SELECT value FROM ? ORDER BY value", value) + assert [tuple(row) for row in cursor.fetchall()] == rows finally: cursor.execute(f"DROP TYPE dbo.[{name}]") db_connection.commit() @@ -358,32 +328,6 @@ def test_implicit_nested_sequences_are_not_tvps(cursor): assert cursor.execute("SELECT 1").fetchone()[0] == 1 -@pytest.mark.asyncio -async def test_async_rejects_streaming_before_consumption(conn_str, tvp_schema): - seen = [] - - def rows(): - seen.append(1) - yield 1, "never" - - async def async_rows(): - seen.append(1) - yield 1, "never" - - connection = await _AsyncConnection.connect(conn_str, autocommit=True) - cursor = connection.cursor() - try: - for source in (rows(), async_rows()): - with pytest.raises(NotSupportedError, match="materialized"): - await cursor.execute(f"EXEC [{tvp_schema}].CountRows ?", items(tvp_schema, source)) - assert seen == [] - await cursor.execute("SELECT 42") - assert (await cursor.fetchone())[0] == 42 - finally: - await cursor.close() - await connection.close() - - @pytest.mark.parametrize( "kwargs,error", [ @@ -468,27 +412,6 @@ def test_sync_temporal_scale_six(cursor, db_connection): db_connection.commit() -@pytest.mark.asyncio -@pytest.mark.parametrize( - "columns,error", - [ - ([("amount", SQL_DECIMAL, 20, 6)], "precision 18 and scale 10"), - ([("stamp", SQL_TYPE_TIMESTAMP, 26, 6)], "scale 7"), - ], -) -async def test_async_native_metadata_limit_is_not_hidden(conn_str, columns, error): - connection = await _AsyncConnection.connect(conn_str, autocommit=True) - cursor = connection.cursor() - try: - await cursor.execute("SELECT 42") - with pytest.raises(ValueError, match=error): - await cursor.execute("SELECT ?", TableValuedParameter("UnusedType", columns, [])) - assert (await cursor.fetchone())[0] == 42 - finally: - await cursor.close() - await connection.close() - - def test_schema_mismatch_and_server_error_recovery(cursor, tvp_schema): with pytest.raises(DatabaseError): cursor.execute( From 1792f1748b72ae3a377d498b86ea9a393ee96ea2 Mon Sep 17 00:00:00 2001 From: Gaurav Sharma Date: Tue, 13 Oct 2026 18:43:33 +0530 Subject: [PATCH 14/15] CHORE: Remove TVP-specific stress pipeline wiring Keep the memory case classified with the existing stress marker without adding a TVP-specific selector or summary entry to the shared pipeline. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- OneBranchPipelines/stress-test-pipeline.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/OneBranchPipelines/stress-test-pipeline.yml b/OneBranchPipelines/stress-test-pipeline.yml index 4cd98d9fe..548f52186 100644 --- a/OneBranchPipelines/stress-test-pipeline.yml +++ b/OneBranchPipelines/stress-test-pipeline.yml @@ -387,7 +387,7 @@ jobs: # Run original stress tests - script: | - python -m pytest tests/test_011_singlethreaded_stress.py "tests/test_028_table_valued_parameters.py::test_isolated_stream_lifecycle[memory]" -v -m "stress" --junitxml=perf-stress-test-results-linux.xml --timeout=1800 --capture=tee-sys + python -m pytest tests/test_011_singlethreaded_stress.py -v -m "stress" --junitxml=perf-stress-test-results-linux.xml --timeout=1800 --capture=tee-sys displayName: 'Run Single-Threaded Stress Tests' env: DB_CONNECTION_STRING: 'Server=localhost;Database=StressTestDB;Uid=testuser;Pwd=$(DB_PASSWORD);TrustServerCertificate=yes' @@ -437,7 +437,6 @@ jobs: echo "Tests executed:" echo " - test_020_multithreaded_stress.py (stress marker)" echo " - test_011_singlethreaded_stress.py (stress marker)" - echo " - test_028_table_valued_parameters.py::test_isolated_stream_lifecycle[memory] (Linux stress)" echo "" echo "See individual job results for details." displayName: 'Print Summary' From 2b928993d4cfa400b030a99f80a06f6678fc520c Mon Sep 17 00:00:00 2001 From: Gaurav Sharma Date: Tue, 13 Oct 2026 20:52:46 +0530 Subject: [PATCH 15/15] FIX: Match TVP column sequence support to the public stub Accept non-empty column sequences and normalize them through the existing validation path, matching the advertised public contract. Preserve row-source semantics and cover list, tuple, and custom sequence inputs. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- mssql_python/table_valued_parameter.py | 6 +++--- tests/test_028_table_valued_parameters.py | 9 +++++++++ 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/mssql_python/table_valued_parameter.py b/mssql_python/table_valued_parameter.py index 69391a986..15488f09d 100644 --- a/mssql_python/table_valued_parameter.py +++ b/mssql_python/table_valued_parameter.py @@ -1,6 +1,6 @@ """Explicit, transport-neutral input values for SQL Server table types.""" -from collections.abc import Iterator +from collections.abc import Iterator, Sequence from datetime import date, datetime, time from decimal import Decimal from uuid import UUID @@ -37,8 +37,8 @@ def __init__(self, type_name, columns, rows, *, schema=None): or len(name.encode("utf-16le")) > 256 ): raise ValueError("TVP names must be unquoted identifiers of 1–128 UTF-16 units") - if not isinstance(columns, (list, tuple)) or not columns: - raise TypeError("TVP columns must be a non-empty list or tuple") + if not isinstance(columns, Sequence) or not columns: + raise TypeError("TVP columns must be a non-empty sequence") normalized = [] for column in columns: if not isinstance(column, tuple) or not 2 <= len(column) <= 4: diff --git a/tests/test_028_table_valued_parameters.py b/tests/test_028_table_valued_parameters.py index 731116c05..6fd0100ff 100644 --- a/tests/test_028_table_valued_parameters.py +++ b/tests/test_028_table_valued_parameters.py @@ -7,6 +7,7 @@ import threading import uuid import weakref +from collections import UserList from decimal import Decimal, localcontext import pytest @@ -348,6 +349,14 @@ def test_explicit_contract_validation(kwargs, error): TableValuedParameter(**arguments) +@pytest.mark.parametrize("container", [list, tuple, UserList]) +def test_column_sequences_match_public_contract(container): + columns = container([("id", SQL_INTEGER), ("label", SQL_WVARCHAR, 100)]) + value = TableValuedParameter("Items", columns, []) + assert value.columns == (("id", SQL_INTEGER, 10, 0), ("label", SQL_WVARCHAR, 100, 0)) + assert columns == container([("id", SQL_INTEGER), ("label", SQL_WVARCHAR, 100)]) + + @pytest.mark.parametrize("bad", [2**31, -(2**31) - 1]) def test_explicit_integer_width_does_not_truncate(cursor, tvp_schema, bad): with pytest.raises(ValueError, match="out of range"):