Skip to content

Commit 59cf6de

Browse files
axellpadillaaxellpadilla
committed
Enhance SQL Server adapter to support safe type expansion for column types
- Implemented column type expansion logic in SQLServerAdapter. - Added support for NVARCHAR and other type promotions in SQLServerColumn. - Introduced tests for integer and numeric type promotions, as well as VARCHAR to NVARCHAR conversions.
1 parent bcf4ac9 commit 59cf6de

File tree

5 files changed

+311
-29
lines changed

5 files changed

+311
-29
lines changed

‎dbt/adapters/sqlserver/sqlserver_adapter.py‎

Lines changed: 62 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,12 @@
1-
from typing import Optional
1+
from typing import List, Optional
22

33
import dbt.exceptions
44
from dbt.adapters.base.impl import ConstraintSupport
5+
from dbt.adapters.cache import _make_ref_key_dict
6+
from dbt.adapters.events.types import ColTypeChange
57
from dbt.adapters.fabric import FabricAdapter
68
from dbt.contracts.graph.nodes import ConstraintType
9+
from dbt_common.events.functions import fire_event
710

811
from dbt.adapters.sqlserver.sqlserver_column import SQLServerColumn
912
from dbt.adapters.sqlserver.sqlserver_connections import SQLServerConnectionManager
@@ -65,3 +68,61 @@ def valid_incremental_strategies(self):
6568
Not used to validate custom strategies defined by end users.
6669
"""
6770
return ["append", "delete+insert", "merge", "microbatch"]
71+
72+
def expand_column_types(self, goal, current):
73+
"""Override to ensure we use the reference column's dtype when constructing the
74+
new column type during an expansion (so NVARCHAR on the goal yields NVARCHAR).
75+
"""
76+
reference_columns = {c.name: c for c in self.get_columns_in_relation(goal)}
77+
78+
target_columns = {c.name: c for c in self.get_columns_in_relation(current)}
79+
80+
for column_name, reference_column in reference_columns.items():
81+
target_column = target_columns.get(column_name)
82+
83+
if target_column is not None and target_column.can_expand_to(
84+
reference_column,
85+
enable_safe_type_expansion=self.behavior.enable_safe_type_expansion,
86+
):
87+
# If the reference column is a string, compute the new type using
88+
# the reference column's instance-level string helper so we
89+
# respect NVARCHAR/NCHAR vs VARCHAR/CHAR correctly. For non-
90+
# string expansions (numeric/integer promotions), use the
91+
# reference column's resolved data_type directly.
92+
if reference_column.is_string():
93+
col_string_size = reference_column.string_size()
94+
new_type = reference_column.string_type_instance(col_string_size)
95+
else:
96+
# For numeric/integer/other type expansions, use the
97+
# reference column's computed data_type (eg. INT,
98+
# DECIMAL(p,s), etc.).
99+
new_type = reference_column.data_type
100+
fire_event(
101+
ColTypeChange(
102+
orig_type=target_column.data_type,
103+
new_type=new_type,
104+
table=_make_ref_key_dict(current),
105+
)
106+
)
107+
108+
self.alter_column_type(current, column_name, new_type)
109+
110+
@property
111+
def _behavior_flags(self) -> List[dict]:
112+
"""Adapter-specific behavior flags. These are merged with project overrides
113+
by the BaseAdapter.behavior machinery.
114+
"""
115+
return [
116+
{
117+
"name": "enable_safe_type_expansion",
118+
"default": False,
119+
"source": "dbt-sqlserver",
120+
"description": (
121+
"Allow the SQL Server adapter to widen column types during schema-expansion. "
122+
"This enables promotions like varchar->nvarchar, "
123+
" bit->tinyint->smallint->int->bigint, "
124+
"and numeric(p,s)->numeric(p2,s2) using alter column."
125+
),
126+
"docs_url": None,
127+
},
128+
]

‎dbt/adapters/sqlserver/sqlserver_column.py‎

Lines changed: 151 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -1,22 +1,154 @@
1-
from dbt.adapters.fabric import FabricColumn
1+
from typing import Any, ClassVar, Dict
22

3+
from dbt.adapters.base import Column
4+
from dbt_common.exceptions import DbtRuntimeError
5+
6+
7+
class SQLServerColumn(Column):
8+
TYPE_LABELS: ClassVar[Dict[str, str]] = {
9+
"STRING": "VARCHAR(8000)",
10+
"VARCHAR": "VARCHAR(8000)",
11+
"CHAR": "CHAR(1)",
12+
"NCHAR": "NCHAR(1)",
13+
"NVARCHAR": "NVARCHAR(4000)",
14+
"TIMESTAMP": "DATETIME2(6)",
15+
"DATETIME2": "DATETIME2(6)",
16+
"DATETIME2(6)": "DATETIME2(6)",
17+
"DATE": "DATE",
18+
"TIME": "TIME(6)",
19+
"FLOAT": "FLOAT",
20+
"REAL": "REAL",
21+
"INT": "INT",
22+
"INTEGER": "INT",
23+
"BIGINT": "BIGINT",
24+
"SMALLINT": "SMALLINT",
25+
"TINYINT": "SMALLINT",
26+
"BIT": "BIT",
27+
"BOOLEAN": "BIT",
28+
"DECIMAL": "DECIMAL",
29+
"NUMERIC": "NUMERIC",
30+
"MONEY": "DECIMAL",
31+
"SMALLMONEY": "DECIMAL",
32+
"UNIQUEIDENTIFIER": "UNIQUEIDENTIFIER",
33+
"VARBINARY": "VARBINARY(MAX)",
34+
"BINARY": "BINARY(1)",
35+
}
36+
37+
@classmethod
38+
def string_type(cls, size: int) -> str:
39+
"""Class-level string_type used by SQLAdapter.expand_column_types.
40+
41+
Return a VARCHAR default for the SQLAdapter path; this keeps behaviour
42+
consistent with the rest of dbt where class-level string_type is
43+
generic and not instance-aware.
44+
"""
45+
return f"varchar({size if size > 0 else '8000'})"
46+
47+
def string_type_instance(self, size: int) -> str:
48+
"""
49+
Instance-level string type selection that respects NVARCHAR/NCHAR.
50+
"""
51+
dtype = (self.dtype or "").lower()
52+
# n types use half the byte size for character count
53+
if dtype == "nvarchar":
54+
return f"nvarchar({size//2 if size > 0 else '4000'})"
55+
if dtype == "nchar":
56+
return f"nchar({size//2 if size > 1 else '1'})"
57+
# default to varchar/char behaviour
58+
return f"varchar({size if size > 0 else '8000'})"
59+
60+
def literal(self, value: Any) -> str:
61+
return "cast('{}' as {})".format(value, self.data_type)
62+
63+
@property
64+
def data_type(self) -> str:
65+
# Always enforce datetime2 precision
66+
if self.dtype.lower() == "datetime2":
67+
return "datetime2(6)"
68+
if self.is_string():
69+
return self.string_type_instance(self.string_size())
70+
elif self.is_numeric():
71+
return self.numeric_type(self.dtype, self.numeric_precision, self.numeric_scale)
72+
else:
73+
return self.dtype
74+
75+
def is_string(self) -> bool:
76+
return self.dtype.lower() in ["varchar", "char", "nvarchar", "nchar"]
77+
78+
def is_number(self):
79+
return any([self.is_integer(), self.is_numeric(), self.is_float()])
80+
81+
def is_float(self):
82+
return self.dtype.lower() in ["float", "real"]
383

4-
class SQLServerColumn(FabricColumn):
584
def is_integer(self) -> bool:
6-
return self.dtype.lower() in [
7-
# real types
8-
"smallint",
9-
"integer",
10-
"bigint",
11-
"smallserial",
12-
"serial",
13-
"bigserial",
14-
# aliases
15-
"int2",
16-
"int4",
17-
"int8",
18-
"serial2",
19-
"serial4",
20-
"serial8",
21-
"int",
22-
]
85+
# Treat BIT as an integer-like type so it participates in integer
86+
# promotions (bit -> tinyint -> smallint -> int -> bigint).
87+
return self.dtype.lower() in ["int", "integer", "bigint", "smallint", "tinyint", "bit"]
88+
89+
def is_numeric(self) -> bool:
90+
return self.dtype.lower() in ["numeric", "decimal", "money", "smallmoney"]
91+
92+
def string_size(self) -> int:
93+
if not self.is_string():
94+
raise DbtRuntimeError("Called string_size() on non-string field!")
95+
if self.char_size is None:
96+
return 8000
97+
else:
98+
return int(self.char_size)
99+
100+
def can_expand_to(
101+
self, other_column: Column, enable_safe_type_expansion: bool = False
102+
) -> bool:
103+
# If both are strings, allow size-based expansion regardless of the
104+
# feature flag. Only allow family changes (VARCHAR -> NVARCHAR) when
105+
# `enable_safe_type_expansion` is set by the adapter.
106+
self_dtype = self.dtype.lower()
107+
other_dtype = other_column.dtype.lower()
108+
if self.is_string() and other_column.is_string():
109+
self_size = self.string_size()
110+
other_size = other_column.string_size()
111+
112+
if other_size > self_size and self_dtype == other_dtype:
113+
return True
114+
115+
# Allow safe conversions across the CHAR/VARCHAR -> NCHAR/NVARCHAR family
116+
# only when the feature flag is enabled. Do NOT allow shrinking
117+
# conversions or NVARCHAR -> VARCHAR.
118+
if self_dtype in ("varchar", "char") and other_dtype in ("nvarchar", "nchar"):
119+
# allow when target has at least the same character capacity
120+
if other_size >= self_size and enable_safe_type_expansion:
121+
return True
122+
123+
# If none of the string rules matched, we can't expand.
124+
return False
125+
126+
# If we reach here, at least one side is not a string. Apply integer/
127+
# numeric promotion logic only if the adapter has enabled type expansion.
128+
if not enable_safe_type_expansion or not self.is_number() or not other_column.is_number():
129+
return False
130+
131+
# Integer family promotions (tinyint -> smallint -> int -> bigint)
132+
int_family = ("bit", "tinyint", "smallint", "int", "bigint")
133+
if self_dtype in int_family and other_dtype in int_family:
134+
if int_family.index(other_dtype) > int_family.index(self_dtype):
135+
return True
136+
137+
self_prec = int(self.numeric_precision or 0)
138+
other_prec = int(other_column.numeric_precision or 0)
139+
# Integer -> numeric/decimal is a safe widening (integers fit in numerics).
140+
if self.is_integer() and other_column.is_numeric() and other_prec > self_prec:
141+
return True
142+
143+
# Numeric/Decimal promotions: allow when target precision >= source precision
144+
# and target scale >= source scale (so we don't lose fractional digits).
145+
if self.is_numeric() and other_column.is_numeric():
146+
# Access precision/scale directly from columns. Fall back to 0 when missing.
147+
self_scale = int(self.numeric_scale or 0)
148+
other_scale = int(other_column.numeric_scale or 0)
149+
150+
if other_prec >= self_prec and other_scale >= self_scale:
151+
if other_prec > self_prec or other_scale > self_scale:
152+
return True
153+
154+
return False

‎tests/functional/adapter/dbt/test_column_types.py‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -88,7 +88,8 @@
8888
CAST(5.0 AS float) as double_col,
8989
CAST(6.0 AS numeric) as numeric_col,
9090
CAST(7 AS varchar(20)) as text_col,
91-
CAST(8 AS varchar(20)) as varchar_col
91+
CAST(8 AS varchar(20)) as varchar_col,
92+
cast(9 as nvarchar(20)) as nvarchar_col
9293
"""
9394

9495
schema_yml = """
@@ -106,6 +107,7 @@
106107
numeric_col: ['numeric', 'number']
107108
text_col: ['string', 'not number']
108109
varchar_col: ['string', 'not number']
110+
nvarchar_col: ['string', 'not number']
109111
""" # noqa
110112

111113

‎tests/functional/adapter/dbt/test_incremental.py‎

Lines changed: 47 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -3,10 +3,7 @@
33
from dbt.tests.adapter.incremental.test_incremental_on_schema_change import (
44
BaseIncrementalOnSchemaChange,
55
)
6-
from dbt.tests.adapter.incremental.test_incremental_predicates import (
7-
TestIncrementalPredicatesDeleteInsert,
8-
TestPredicatesDeleteInsert,
9-
)
6+
from dbt.tests.util import run_dbt, write_file
107

118
_MODELS__INCREMENTAL_IGNORE_SQLServer = """
129
{{
@@ -92,9 +89,51 @@ def models(self):
9289
}
9390

9491

95-
class TestIncrementalPredicatesDeleteInsert(TestIncrementalPredicatesDeleteInsert):
96-
pass
92+
_INCREMENTAL__WIDEN_TYPES_SQLServer = """
93+
{{
94+
config(
95+
materialized='incremental',
96+
unique_key='id',
97+
on_schema_change='append_new_columns'
98+
)
99+
}}
100+
101+
{% if is_incremental() %}
102+
-- incremental branch: uses larger types and values that would fail if table types were not widened
103+
select
104+
2 as id,
105+
cast(40000 as int) as num_int,
106+
cast('abcdef' as nvarchar(10)) as field1,
107+
cast(100.25 as decimal(10,4)) as num_decimal,
108+
cast(999999999999998.9999 as decimal(20,4)) as num_money
109+
{% else %}
110+
-- full-refresh branch: creates the table with smaller types
111+
select
112+
1 as id,
113+
cast(1 as smallint) as num_int,
114+
cast('abc' as varchar(5)) as field1,
115+
cast(10.5 as decimal(5,2)) as num_decimal,
116+
cast(1240.14 as money) as num_money
117+
{% endif %}
118+
"""
119+
120+
121+
class TestIncrementalOnSchemaChangeExpands:
122+
@pytest.fixture(scope="class")
123+
def project_config_update(self):
124+
return {"flags": {"enable_safe_type_expansion": True}}
125+
126+
def test_run_incremental_widen_types(self, project):
127+
"""Full-refresh to create small types, then incremental to widen types."""
128+
write_file(_INCREMENTAL__WIDEN_TYPES_SQLServer, "models", "incremental_change_widen.sql")
129+
130+
# Full-refresh to create table with smallint and varchar(5)
131+
run_dbt(
132+
["run", "--models", "incremental_change_widen", "--full-refresh"]
133+
) # creates small types
97134

135+
# Run again to trigger incremental insert which requires widened types
136+
# incremental branch inserts larger values
137+
run_dbt(["run", "--models", "incremental_change_widen"])
98138

99-
class TestPredicatesDeleteInsert(TestPredicatesDeleteInsert):
100-
pass
139+
return True

‎tests/unit/adapters/mssql/test_can_expand_to.py‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
import pytest
2+
3+
from dbt.adapters.sqlserver.sqlserver_column import SQLServerColumn
4+
5+
6+
def col_kwargs(dtype, char_size=None, numeric_precision=0, numeric_scale=0):
7+
return {
8+
"column": "c",
9+
"dtype": dtype,
10+
"char_size": char_size,
11+
"numeric_precision": numeric_precision,
12+
"numeric_scale": numeric_scale,
13+
}
14+
15+
16+
@pytest.mark.parametrize(
17+
"src_kwargs,tgt_kwargs,expect_with_flag,expect_without_flag",
18+
[
19+
# Integer family promotions require the feature flag
20+
(col_kwargs("int"), col_kwargs("bigint"), True, False),
21+
(col_kwargs("bit"), col_kwargs("tinyint"), True, False),
22+
# Integer -> numeric widening requires the feature flag
23+
(col_kwargs("int"), col_kwargs("numeric", numeric_precision=10), True, False),
24+
(col_kwargs("bit"), col_kwargs("numeric", numeric_precision=5), True, False),
25+
# Numeric/decimal promotions: precision/scale must increase; flag required
26+
(
27+
col_kwargs("numeric", numeric_precision=10, numeric_scale=2),
28+
col_kwargs("numeric", numeric_precision=12, numeric_scale=4),
29+
True,
30+
False,
31+
),
32+
(
33+
col_kwargs("numeric", numeric_precision=10, numeric_scale=2),
34+
col_kwargs("numeric", numeric_precision=12, numeric_scale=1),
35+
False,
36+
False,
37+
),
38+
# String family change (VARCHAR -> NVARCHAR) is only allowed when the
39+
# feature flag is set and capacity is sufficient
40+
(col_kwargs("varchar", char_size=10), col_kwargs("nvarchar", char_size=10), True, False),
41+
],
42+
)
43+
def test_can_expand_parametrized(src_kwargs, tgt_kwargs, expect_with_flag, expect_without_flag):
44+
src = SQLServerColumn(**src_kwargs)
45+
tgt = SQLServerColumn(**tgt_kwargs)
46+
47+
assert src.can_expand_to(tgt, enable_safe_type_expansion=True) is expect_with_flag
48+
assert src.can_expand_to(tgt, enable_safe_type_expansion=False) is expect_without_flag

0 commit comments

Comments
 (0)