Skip to content

Commit db746a8

Browse files
bewithgauravbewithgaurav
committed
Merge branch 'main' of https://github.com/microsoft/mssql-python into bewithgaurav/fix-bulkcopy-entra-auth-cleanup
2 parents 2bbe919 + 590af57 commit db746a8

5 files changed

Lines changed: 799 additions & 50 deletions

File tree

‎mssql_python/constants.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -118,6 +118,7 @@ class ConstantsDDBC(Enum):
118118
SQL_DATETIMEOFFSET = -155
119119
SQL_SS_TIME2 = -154
120120
SQL_SS_XML = -152
121+
SQL_SS_VARIANT = -150
121122
SQL_C_SS_TIMESTAMPOFFSET = 0x4001
122123
SQL_SCOPE_CURROW = 0
123124
SQL_BEST_ROWID = 1
@@ -376,6 +377,7 @@ def get_valid_types(cls) -> set:
376377
ConstantsDDBC.SQL_SS_XML.value,
377378
ConstantsDDBC.SQL_GUID.value,
378379
ConstantsDDBC.SQL_SS_UDT.value,
380+
ConstantsDDBC.SQL_SS_VARIANT.value,
379381
}
380382

381383
# Could also add category methods for convenience

‎mssql_python/cursor.py‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -396,7 +396,7 @@ def _map_sql_type( # pylint: disable=too-many-arguments,too-many-positional-arg
396396
if param is None:
397397
logger.debug("_map_sql_type: NULL parameter - index=%d", i)
398398
return (
399-
ddbc_sql_const.SQL_VARCHAR.value,
399+
ddbc_sql_const.SQL_UNKNOWN_TYPE.value,
400400
ddbc_sql_const.SQL_C_DEFAULT.value,
401401
1,
402402
0,
@@ -883,6 +883,7 @@ def _get_c_type_for_sql_type(self, sql_type: int) -> int:
883883
# Other types
884884
ddbc_sql_const.SQL_GUID.value: ddbc_sql_const.SQL_C_GUID.value,
885885
ddbc_sql_const.SQL_SS_XML.value: ddbc_sql_const.SQL_C_WCHAR.value,
886+
ddbc_sql_const.SQL_SS_VARIANT.value: ddbc_sql_const.SQL_C_BINARY.value,
886887
}
887888
return sql_to_c_type.get(sql_type, ddbc_sql_const.SQL_C_DEFAULT.value)
888889

@@ -2208,6 +2209,16 @@ def executemany( # pylint: disable=too-many-locals,too-many-branches,too-many-s
22082209
min_val=min_val,
22092210
max_val=max_val,
22102211
)
2212+
2213+
# For executemany with all-NULL columns, SQL_UNKNOWN_TYPE doesn't work
2214+
# with array binding. Fall back to SQL_VARCHAR as a safe default.
2215+
if (
2216+
sample_value is None
2217+
and paraminfo.paramSQLType == ddbc_sql_const.SQL_UNKNOWN_TYPE.value
2218+
):
2219+
paraminfo.paramSQLType = ddbc_sql_const.SQL_VARCHAR.value
2220+
paraminfo.columnSize = 1
2221+
22112222
# Special handling for binary data in auto-detected types
22122223
if paraminfo.paramSQLType in (
22132224
ddbc_sql_const.SQL_BINARY.value,

‎mssql_python/pybind/ddbc_bindings.cpp‎

Lines changed: 161 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,19 @@
2828
#define SQL_MAX_NUMERIC_LEN 16
2929
#define SQL_SS_XML (-152)
3030
#define SQL_SS_UDT (-151)
31+
#define SQL_SS_VARIANT (-150)
32+
#define SQL_CA_SS_VARIANT_TYPE (1215)
33+
#ifndef SQL_C_DATE
34+
#define SQL_C_DATE (9)
35+
#endif
36+
#ifndef SQL_C_TIME
37+
#define SQL_C_TIME (10)
38+
#endif
39+
#ifndef SQL_C_TIMESTAMP
40+
#define SQL_C_TIMESTAMP (11)
41+
#endif
42+
// SQL Server-specific variant TIME type code
43+
#define SQL_SS_VARIANT_TIME (16384)
3144

3245
#define STRINGIFY_FOR_CASE(x) \
3346
case x: \
@@ -471,14 +484,21 @@ SQLRETURN BindParameters(SQLHANDLE hStmt, const py::list& params,
471484
hStmt, static_cast<SQLUSMALLINT>(paramIndex + 1), &describedType,
472485
&describedSize, &describedDigits, &nullable);
473486
if (!SQL_SUCCEEDED(rc)) {
474-
LOG("BindParameters: SQLDescribeParam failed for "
475-
"param[%d] (NULL parameter) - SQLRETURN=%d",
476-
paramIndex, rc);
477-
return rc;
487+
// SQLDescribeParam can fail for generic SELECT statements where
488+
// no table column is referenced. Fall back to SQL_VARCHAR as a safe
489+
// default.
490+
LOG_WARNING("BindParameters: SQLDescribeParam failed for "
491+
"param[%d] (NULL parameter) - SQLRETURN=%d, falling back to "
492+
"SQL_VARCHAR",
493+
paramIndex, rc);
494+
sqlType = SQL_VARCHAR;
495+
columnSize = 1;
496+
decimalDigits = 0;
497+
} else {
498+
sqlType = describedType;
499+
columnSize = describedSize;
500+
decimalDigits = describedDigits;
478501
}
479-
sqlType = describedType;
480-
columnSize = describedSize;
481-
decimalDigits = describedDigits;
482502
}
483503
dataPtr = nullptr;
484504
strLenOrIndPtr = AllocateParamBuffer<SQLLEN>(paramBuffers);
@@ -2907,6 +2927,67 @@ py::object FetchLobColumnData(SQLHSTMT hStmt, SQLUSMALLINT colIndex, SQLSMALLINT
29072927
}
29082928
}
29092929

2930+
// Helper function to map sql_variant's underlying C type to SQL data type
2931+
// This allows sql_variant to reuse existing fetch logic for each data type
2932+
SQLSMALLINT MapVariantCTypeToSQLType(SQLLEN variantCType) {
2933+
switch (variantCType) {
2934+
case SQL_C_SLONG:
2935+
case SQL_C_LONG:
2936+
return SQL_INTEGER;
2937+
case SQL_C_SSHORT:
2938+
case SQL_C_SHORT:
2939+
return SQL_SMALLINT;
2940+
case SQL_C_SBIGINT:
2941+
return SQL_BIGINT;
2942+
case SQL_C_FLOAT:
2943+
return SQL_REAL;
2944+
case SQL_C_DOUBLE:
2945+
return SQL_DOUBLE;
2946+
case SQL_C_BIT:
2947+
return SQL_BIT;
2948+
case SQL_C_CHAR:
2949+
return SQL_VARCHAR;
2950+
case SQL_C_WCHAR:
2951+
return SQL_WVARCHAR;
2952+
case SQL_C_DATE:
2953+
case SQL_C_TYPE_DATE:
2954+
return SQL_TYPE_DATE;
2955+
case SQL_C_TIME:
2956+
case SQL_C_TYPE_TIME:
2957+
case SQL_SS_VARIANT_TIME:
2958+
return SQL_TYPE_TIME;
2959+
case SQL_C_TIMESTAMP:
2960+
case SQL_C_TYPE_TIMESTAMP:
2961+
return SQL_TYPE_TIMESTAMP;
2962+
case SQL_C_BINARY:
2963+
return SQL_VARBINARY;
2964+
case SQL_C_GUID:
2965+
return SQL_GUID;
2966+
case SQL_C_NUMERIC:
2967+
return SQL_NUMERIC;
2968+
case SQL_C_TINYINT:
2969+
case SQL_C_UTINYINT:
2970+
case SQL_C_STINYINT:
2971+
return SQL_TINYINT;
2972+
default:
2973+
// Unknown C type code - fallback to WVARCHAR for string conversion
2974+
// Note: SQL Server enforces sql_variant restrictions at INSERT time, preventing
2975+
// invalid types (text, ntext, image, timestamp, xml, MAX types, nested variants,
2976+
// spatial types, hierarchyid, UDTs) from being stored. By the time we fetch data,
2977+
// only valid base types exist. This default handles unmapped/future type codes.
2978+
return SQL_WVARCHAR;
2979+
}
2980+
}
2981+
2982+
// Helper function to check if a column requires SQLGetData streaming (LOB or sql_variant)
2983+
static inline bool IsLobOrVariantColumn(SQLSMALLINT dataType, SQLULEN columnSize) {
2984+
return dataType == SQL_SS_VARIANT ||
2985+
((dataType == SQL_WVARCHAR || dataType == SQL_WLONGVARCHAR || dataType == SQL_VARCHAR ||
2986+
dataType == SQL_LONGVARCHAR || dataType == SQL_VARBINARY ||
2987+
dataType == SQL_LONGVARBINARY || dataType == SQL_SS_XML || dataType == SQL_SS_UDT) &&
2988+
(columnSize == 0 || columnSize == SQL_NO_TOTAL || columnSize > SQL_MAX_LOB_SIZE));
2989+
}
2990+
29102991
// Helper function to retrieve column data
29112992
SQLRETURN SQLGetData_wrap(SqlHandlePtr StatementHandle, SQLUSMALLINT colCount, py::list& row,
29122993
const std::string& charEncoding = "utf-8",
@@ -2945,7 +3026,42 @@ SQLRETURN SQLGetData_wrap(SqlHandlePtr StatementHandle, SQLUSMALLINT colCount, p
29453026
continue;
29463027
}
29473028

2948-
switch (dataType) {
3029+
// Preprocess sql_variant: detect underlying type to route to correct conversion logic
3030+
SQLSMALLINT effectiveDataType = dataType;
3031+
if (dataType == SQL_SS_VARIANT) {
3032+
// For sql_variant, we MUST call SQLGetData with SQL_C_BINARY (NULL buffer, len=0)
3033+
// first. This serves two purposes:
3034+
// 1. Detects NULL values via the indicator parameter
3035+
// 2. Initializes the variant metadata in the ODBC driver, which is required for
3036+
// SQLColAttribute(SQL_CA_SS_VARIANT_TYPE) to return the correct underlying C type.
3037+
// Without this probe call, SQLColAttribute returns incorrect type codes.
3038+
SQLLEN indicator;
3039+
ret = SQLGetData_ptr(hStmt, i, SQL_C_BINARY, NULL, 0, &indicator);
3040+
if (!SQL_SUCCEEDED(ret)) {
3041+
LOG_ERROR("SQLGetData: Failed to probe sql_variant column %d - SQLRETURN=%d", i,
3042+
ret);
3043+
row.append(py::none());
3044+
continue;
3045+
}
3046+
if (indicator == SQL_NULL_DATA) {
3047+
row.append(py::none());
3048+
continue;
3049+
}
3050+
// Now retrieve the underlying C type
3051+
SQLLEN variantCType = 0;
3052+
ret =
3053+
SQLColAttribute_ptr(hStmt, i, SQL_CA_SS_VARIANT_TYPE, NULL, 0, NULL, &variantCType);
3054+
if (!SQL_SUCCEEDED(ret)) {
3055+
LOG_ERROR("SQLGetData: Failed to get sql_variant underlying type for column %d", i);
3056+
row.append(py::none());
3057+
continue;
3058+
}
3059+
effectiveDataType = MapVariantCTypeToSQLType(variantCType);
3060+
LOG("SQLGetData: sql_variant column %d has variantCType=%ld, mapped to SQL type %d", i,
3061+
(long)variantCType, effectiveDataType);
3062+
}
3063+
3064+
switch (effectiveDataType) {
29493065
case SQL_CHAR:
29503066
case SQL_VARCHAR:
29513067
case SQL_LONGVARCHAR: {
@@ -4048,7 +4164,8 @@ size_t calculateRowSize(py::list& columnNames, SQLUSMALLINT numCols) {
40484164
break;
40494165
case SQL_SS_UDT:
40504166
rowSize += (static_cast<SQLLEN>(columnSize) == SQL_NO_TOTAL || columnSize == 0)
4051-
? SQL_MAX_LOB_SIZE : columnSize;
4167+
? SQL_MAX_LOB_SIZE
4168+
: columnSize;
40524169
break;
40534170
case SQL_BINARY:
40544171
case SQL_VARBINARY:
@@ -4110,11 +4227,7 @@ SQLRETURN FetchMany_wrap(SqlHandlePtr StatementHandle, py::list& rows, int fetch
41104227
SQLSMALLINT dataType = colMeta["DataType"].cast<SQLSMALLINT>();
41114228
SQLULEN columnSize = colMeta["ColumnSize"].cast<SQLULEN>();
41124229

4113-
if ((dataType == SQL_WVARCHAR || dataType == SQL_WLONGVARCHAR || dataType == SQL_VARCHAR ||
4114-
dataType == SQL_LONGVARCHAR || dataType == SQL_VARBINARY ||
4115-
dataType == SQL_LONGVARBINARY || dataType == SQL_SS_XML ||
4116-
dataType == SQL_SS_UDT) &&
4117-
(columnSize == 0 || columnSize == SQL_NO_TOTAL || columnSize > SQL_MAX_LOB_SIZE)) {
4230+
if (IsLobOrVariantColumn(dataType, columnSize)) {
41184231
lobColumns.push_back(i + 1); // 1-based
41194232
}
41204233
}
@@ -4204,6 +4317,40 @@ SQLRETURN FetchAll_wrap(SqlHandlePtr StatementHandle, py::list& rows,
42044317
return ret;
42054318
}
42064319

4320+
std::vector<SQLUSMALLINT> lobColumns;
4321+
for (SQLSMALLINT i = 0; i < numCols; i++) {
4322+
auto colMeta = columnNames[i].cast<py::dict>();
4323+
SQLSMALLINT dataType = colMeta["DataType"].cast<SQLSMALLINT>();
4324+
SQLULEN columnSize = colMeta["ColumnSize"].cast<SQLULEN>();
4325+
4326+
// Detect LOB columns that need SQLGetData streaming
4327+
// sql_variant always uses SQLGetData for native type preservation
4328+
if (IsLobOrVariantColumn(dataType, columnSize)) {
4329+
lobColumns.push_back(i + 1); // 1-based
4330+
}
4331+
}
4332+
4333+
// If we have LOBs → fall back to row-by-row fetch + SQLGetData_wrap
4334+
if (!lobColumns.empty()) {
4335+
LOG("FetchAll_wrap: LOB columns detected (%zu columns), using per-row "
4336+
"SQLGetData path",
4337+
lobColumns.size());
4338+
while (true) {
4339+
ret = SQLFetch_ptr(hStmt);
4340+
if (ret == SQL_NO_DATA)
4341+
break;
4342+
if (!SQL_SUCCEEDED(ret))
4343+
return ret;
4344+
4345+
py::list row;
4346+
SQLGetData_wrap(StatementHandle, numCols, row, charEncoding,
4347+
wcharEncoding); // <-- streams LOBs correctly
4348+
rows.append(row);
4349+
}
4350+
return SQL_SUCCESS;
4351+
}
4352+
4353+
// No LOBs detected - use binding path with batch fetching
42074354
// Define a memory limit (1 GB)
42084355
const size_t memoryLimit = 1ULL * 1024 * 1024 * 1024;
42094356
size_t totalRowSize = calculateRowSize(columnNames, numCols);
@@ -4244,41 +4391,6 @@ SQLRETURN FetchAll_wrap(SqlHandlePtr StatementHandle, py::list& rows,
42444391
}
42454392
LOG("FetchAll_wrap: Fetching data in batch sizes of %d", fetchSize);
42464393

4247-
std::vector<SQLUSMALLINT> lobColumns;
4248-
for (SQLSMALLINT i = 0; i < numCols; i++) {
4249-
auto colMeta = columnNames[i].cast<py::dict>();
4250-
SQLSMALLINT dataType = colMeta["DataType"].cast<SQLSMALLINT>();
4251-
SQLULEN columnSize = colMeta["ColumnSize"].cast<SQLULEN>();
4252-
4253-
if ((dataType == SQL_WVARCHAR || dataType == SQL_WLONGVARCHAR || dataType == SQL_VARCHAR ||
4254-
dataType == SQL_LONGVARCHAR || dataType == SQL_VARBINARY ||
4255-
dataType == SQL_LONGVARBINARY || dataType == SQL_SS_XML ||
4256-
dataType == SQL_SS_UDT) &&
4257-
(columnSize == 0 || columnSize == SQL_NO_TOTAL || columnSize > SQL_MAX_LOB_SIZE)) {
4258-
lobColumns.push_back(i + 1); // 1-based
4259-
}
4260-
}
4261-
4262-
// If we have LOBs → fall back to row-by-row fetch + SQLGetData_wrap
4263-
if (!lobColumns.empty()) {
4264-
LOG("FetchAll_wrap: LOB columns detected (%zu columns), using per-row "
4265-
"SQLGetData path",
4266-
lobColumns.size());
4267-
while (true) {
4268-
ret = SQLFetch_ptr(hStmt);
4269-
if (ret == SQL_NO_DATA)
4270-
break;
4271-
if (!SQL_SUCCEEDED(ret))
4272-
return ret;
4273-
4274-
py::list row;
4275-
SQLGetData_wrap(StatementHandle, numCols, row, charEncoding,
4276-
wcharEncoding); // <-- streams LOBs correctly
4277-
rows.append(row);
4278-
}
4279-
return SQL_SUCCESS;
4280-
}
4281-
42824394
ColumnBuffers buffers(numCols, fetchSize);
42834395

42844396
// Bind columns

‎tests/test_004_cursor.py‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -658,6 +658,23 @@ def test_varbinary_full_capacity(cursor, db_connection):
658658
db_connection.commit()
659659

660660

661+
def test_execute_none_into_varbinary_column(cursor, db_connection):
662+
from mssql_python.constants import ConstantsDDBC
663+
664+
drop_table_if_exists(cursor, "#test_varbinary_null")
665+
try:
666+
cursor.execute("CREATE TABLE #test_varbinary_null (data VARBINARY(100))")
667+
db_connection.commit()
668+
cursor.setinputsizes([(ConstantsDDBC.SQL_VARBINARY.value, 100, 0)])
669+
cursor.execute("INSERT INTO #test_varbinary_null (data) VALUES (?)", None)
670+
db_connection.commit()
671+
cursor.execute("SELECT data FROM #test_varbinary_null")
672+
row = cursor.fetchone()
673+
assert row[0] is None
674+
finally:
675+
drop_table_if_exists(cursor, "#test_varbinary_null")
676+
677+
661678
def test_varbinary_max(cursor, db_connection):
662679
"""Test SQL_VARBINARY with MAX length"""
663680
try:

0 commit comments

Comments
 (0)