diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index 625bf62150..a7788df73d 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -114,7 +114,7 @@ Formatting settings for the `sqlmesh format` command and UI. | `normalize` | Whether to normalize SQL (Default: False) | boolean | N | | `pad` | The number of spaces to use for padding (Default: 2) | int | N | | `indent` | The number of spaces to use for indentation (Default: 2) | int | N | -| `normalize_functions` | Whether to normalize function names. Supported values are: 'upper' and 'lower' (Default: None) | string | N | +| `normalize_functions` | How to normalize function name casing. `false` (default) preserves the casing of custom and audit function names as written; `"upper"` uppercases all function names; `"lower"` lowercases all function names; `true` defers to SQLGlot's generator default and uppercases all function names including custom ones; `null` (or omitting the key) is excluded during serialization and therefore takes the same `false` default path — it does **not** defer to SQLGlot's generator default. Note: SQLGlot built-in function names may still be canonicalized by the parser regardless of this setting. | string \| boolean \| null | N | | `leading_comma` | Whether to use leading commas (Default: False) | boolean | N | | `max_text_width` | The maximum text width in a segment before creating new lines (Default: 80) | int | N | | `append_newline` | Whether to append a newline to the end of the file (Default: False) | boolean | N | diff --git a/sqlmesh/core/config/format.py b/sqlmesh/core/config/format.py index 8730425d2e..5ec6da47cd 100644 --- a/sqlmesh/core/config/format.py +++ b/sqlmesh/core/config/format.py @@ -12,7 +12,21 @@ class FormatConfig(BaseConfig): normalize: Whether to normalize the SQL code or not. pad: The number of spaces to use for padding. indent: The number of spaces to use for indentation. - normalize_functions: Whether or not to normalize all function names. Possible values are: 'upper', 'lower' + normalize_functions: How to normalize function name casing. + + * ``False`` (default) — preserves the original spelling of custom and audit + function names. SQLGlot built-in functions (e.g. ``COUNT``, ``SUM``) may + still be uppercased because the parser discards the original token. + * ``"upper"`` — uppercases all function names, including custom audit + references. + * ``"lower"`` — lowercases all function names, including built-in ones. + * ``True`` — defers to SQLGlot's generator default, which uppercases all + function names including custom ones. + * ``None`` — excluded from the serialized generator options by Pydantic's + ``exclude_none`` behaviour, so ``format_model_expressions`` falls back to + its own ``False`` default. Setting this in YAML as ``null`` or omitting + the key is therefore equivalent to ``false``; it does **not** defer to + SQLGlot's generator default the way ``True`` does. leading_comma: Whether to use leading commas or not. max_text_width: The maximum text width in a segment before creating new lines. append_newline: Whether to append a newline to the end of the file or not. @@ -22,7 +36,7 @@ class FormatConfig(BaseConfig): normalize: bool = False pad: int = 2 indent: int = 2 - normalize_functions: t.Optional[str] = None + normalize_functions: t.Union[str, bool, None] = False leading_comma: bool = False max_text_width: int = 80 append_newline: bool = False diff --git a/sqlmesh/core/dialect.py b/sqlmesh/core/dialect.py index af550378b4..67918b6d14 100644 --- a/sqlmesh/core/dialect.py +++ b/sqlmesh/core/dialect.py @@ -790,6 +790,7 @@ def format_model_expressions( expressions: t.List[exp.Expr], dialect: t.Optional[str] = None, rewrite_casts: bool = True, + normalize_functions: t.Union[str, bool, None] = False, **kwargs: t.Any, ) -> str: """Format a model's expressions into a standardized format. @@ -798,6 +799,21 @@ def format_model_expressions( expressions: The model's expressions, must be at least model def + query. dialect: The dialect to render the expressions as. rewrite_casts: Whether to rewrite all casts to use the :: syntax. + normalize_functions: How to normalize function name casing. + + * ``False`` (default) — preserves the original spelling of custom and audit + function names. SQLGlot built-in functions may still canonicalize because + the parser discards the original token. + * ``"upper"`` — uppercases all function names including custom audit + references. + * ``"lower"`` — lowercases all function names including built-ins. + * ``True`` — defers to SQLGlot's generator default (uppercase). + * ``None`` — passes ``None`` directly to the SQLGlot generator, which + defers to SQLGlot's own default (typically uppercase, but may vary by + dialect). Note: this is the **direct generator API** behaviour. When + called via ``FormatConfig``, ``None`` is excluded by Pydantic's + ``exclude_none`` serialization and this function receives its own ``False`` + default instead — so the two paths are not equivalent. **kwargs: Additional keyword arguments to pass to the sql generator. Returns: @@ -807,7 +823,9 @@ def format_model_expressions( # Meta expressions (MODEL/AUDIT/METRIC) are SQLMesh DDL, not standard SQL, # so they must never be transpiled to the target dialect (e.g. tsql would # rewrite a boolean property like `allow_partials TRUE` to `(1 = 1)`). - return expressions[0].sql(pretty=True, dialect=None) + return expressions[0].sql( + pretty=True, dialect=None, normalize_functions=normalize_functions + ) if rewrite_casts: @@ -844,6 +862,7 @@ def cast_to_colon(node: exp.Expr) -> exp.Expr: expression.sql( pretty=True, dialect=None if is_meta_expression(expression) else dialect, + normalize_functions=normalize_functions, **kwargs, ) for expression in expressions diff --git a/tests/core/test_dialect.py b/tests/core/test_dialect.py index 62aacd9eca..e2f1daba3d 100644 --- a/tests/core/test_dialect.py +++ b/tests/core/test_dialect.py @@ -15,6 +15,7 @@ import sqlmesh.core.dialect as d from sqlmesh.core.model import SqlModel, load_sql_based_model from sqlmesh.core.config.connection import DIALECT_TO_TYPE +from sqlmesh.core.config.format import FormatConfig pytestmark = pytest.mark.dialect_isolated @@ -98,7 +99,7 @@ def test_format_model_expressions(): references (a, (b, c) AS d), /* c */ @macro_prop_with_comment(proper := 'foo'), /* k */ audits ARRAY( - NOT_NULL( + not_null( columns = ARRAY( foo_id, foo_normalised, @@ -112,14 +113,14 @@ def test_format_model_expressions(): tier ) ), - UNIQUE_VALUES(columns = ARRAY(foo_id)), - ACCEPTED_RANGE(column = foo_normalised, min_v = 0, max_v = 100), - ACCEPTED_RANGE(column = bar_normalised, min_v = 0, max_v = 100), - ACCEPTED_RANGE(column = total_weight, min_v = 0, max_v = 100), - ACCEPTED_RANGE(column = cumulative_total_weight_share, min_v = 0, max_v = 1), - ACCEPTED_RANGE(column = market_cumulative_total_weight_share, min_v = 0, max_v = 1), - ACCEPTED_VALUES(column = tier, is_in = ARRAY('Tier 1', 'Tier 2', 'Tier 3', 'Long Tail')), - ACCEPTED_VALUES( + unique_values(columns = ARRAY(foo_id)), + accepted_range(column = foo_normalised, min_v = 0, max_v = 100), + accepted_range(column = bar_normalised, min_v = 0, max_v = 100), + accepted_range(column = total_weight, min_v = 0, max_v = 100), + accepted_range(column = cumulative_total_weight_share, min_v = 0, max_v = 1), + accepted_range(column = market_cumulative_total_weight_share, min_v = 0, max_v = 1), + accepted_values(column = tier, is_in = ARRAY('Tier 1', 'Tier 2', 'Tier 3', 'Long Tail')), + accepted_values( column = total_weight_decile, is_in = ARRAY( 'Decile_01', @@ -341,6 +342,222 @@ def test_format_model_expressions(): ) +def test_format_model_expressions_normalize_functions(): + """Regression: formatter function-name casing behavior. + + Approved behavior: + - Default (``normalize_functions=False``): custom/audit function names + (stored as strings in the AST) are preserved with their original casing. + SQLGlot built-in functions like COUNT/SUM are always output in their + canonical uppercase form because the parser discards the original spelling. + - ``normalize_functions="upper"``: both audit references and query functions + are uppercased. + - ``normalize_functions="lower"``: both audit references and query functions + are lowercased. + + The fix also covers the single-meta-expression early-return path in + ``format_model_expressions``; assertions at the end of this test exercise + that path to prevent regression. + """ + expressions = parse( + """ + MODEL ( + name x, + audits ( + unique_combination_of_columns(columns := (id)), + not_null(columns := (id)) + ) + ); + + SELECT SUM(id), count(id) FROM foo; + """ + ) + + # Default: audit references preserved lowercase; COUNT/SUM canonicalized uppercase. + assert ( + format_model_expressions(expressions) + == """MODEL ( + name x, + audits ( + unique_combination_of_columns(columns := ( + id + )), + not_null(columns := ( + id + )) + ) +); + +SELECT + SUM(id), + COUNT(id) +FROM foo""" + ) + + # "upper": audit references uppercased; query functions uppercased. + assert ( + format_model_expressions(expressions, normalize_functions="upper") + == """MODEL ( + name x, + audits ( + UNIQUE_COMBINATION_OF_COLUMNS(columns := ( + id + )), + NOT_NULL(columns := ( + id + )) + ) +); + +SELECT + SUM(id), + COUNT(id) +FROM foo""" + ) + + # "lower": audit references preserved lowercase (already lower); query functions lowercased. + assert ( + format_model_expressions(expressions, normalize_functions="lower") + == """MODEL ( + name x, + audits ( + unique_combination_of_columns(columns := ( + id + )), + not_null(columns := ( + id + )) + ) +); + +SELECT + sum(id), + count(id) +FROM foo""" + ) + + # None: explicit deferral to SQLGlot default → custom/audit names uppercased, + # just like "upper". This is distinct from False (preserve) and must be tested + # explicitly because None used to be indistinguishable from the missing kwarg. + assert ( + format_model_expressions(expressions, normalize_functions=None) + == """MODEL ( + name x, + audits ( + UNIQUE_COMBINATION_OF_COLUMNS(columns := ( + id + )), + NOT_NULL(columns := ( + id + )) + ) +); + +SELECT + SUM(id), + COUNT(id) +FROM foo""" + ) + + # Single-meta-expression path: normalize_functions must be forwarded. + # Without the fix, this path ignored normalize_functions entirely. + single_model = parse( + """ + MODEL ( + name x, + audits ( + unique_combination_of_columns(columns := (id)), + not_null(columns := (id)) + ) + ); + """ + ) + + assert ( + format_model_expressions(single_model, normalize_functions="upper") + == """MODEL ( + name x, + audits ( + UNIQUE_COMBINATION_OF_COLUMNS(columns := ( + id + )), + NOT_NULL(columns := ( + id + )) + ) +)""" + ) + + assert ( + format_model_expressions(single_model) + == """MODEL ( + name x, + audits ( + unique_combination_of_columns(columns := ( + id + )), + not_null(columns := ( + id + )) + ) +)""" + ) + + # Single-meta path, None: custom audit names are uppercased (explicit SQLGlot default deferral). + assert ( + format_model_expressions(single_model, normalize_functions=None) + == """MODEL ( + name x, + audits ( + UNIQUE_COMBINATION_OF_COLUMNS(columns := ( + id + )), + NOT_NULL(columns := ( + id + )) + ) +)""" + ) + + +def test_format_config_normalize_functions_false(): + config = FormatConfig(normalize_functions=False) + + assert config.normalize_functions is False + assert config.generator_options["normalize_functions"] is False + + +def test_format_config_normalize_functions_none(): + """FormatConfig(normalize_functions=None) must be accepted but excluded from + generator_options by Pydantic's exclude_none serialization. The config-layer + null therefore takes the False-default path in format_model_expressions rather + than deferring to SQLGlot's generator default the way True does. + """ + config = FormatConfig(normalize_functions=None) + + assert config.normalize_functions is None + # None is excluded by PydanticModel.dict(exclude_none=True), so the key must + # be absent from generator_options — format_model_expressions will use False. + assert "normalize_functions" not in config.generator_options + + # Confirm the False-default behaviour: custom audit names must be preserved. + expressions = parse( + """ + MODEL ( + name x, + audits ( + unique_combination_of_columns(columns := (id)), + not_null(columns := (id)) + ) + ); + SELECT id FROM foo + """ + ) + result = format_model_expressions(expressions, **config.generator_options) + assert "unique_combination_of_columns" in result + assert "not_null" in result + + def test_macro_format(): assert parse_one("@EACH(ARRAY(1,2), x -> x)").sql() == "@EACH(ARRAY(1, 2), x -> x)" assert parse_one("INTERVAL @x DAY").sql() == "INTERVAL @x DAY"