Edit on GitHub

Warns dbt users about potential diffs due to inclusion of {{ config(...) }} blocks in model SQL.

Prior to this fix, SQLMesh wasn't including the {{ config(...) }} block in the model's SQL payload when processing dbt models. Now these config blocks are properly included in the raw SQL, which may cause diffs to appear for existing dbt models even though the actual SQL logic hasn't changed.

This is a one-time diff that will appear after upgrading, and applying a plan will resolve it.

 1"""
 2Warns dbt users about potential diffs due to inclusion of {{ config(...) }} blocks in model SQL.
 3
 4Prior to this fix, SQLMesh wasn't including the {{ config(...) }} block in the model's SQL payload
 5when processing dbt models. Now these config blocks are properly included in the raw SQL, which
 6may cause diffs to appear for existing dbt models even though the actual SQL logic hasn't changed.
 7
 8This is a one-time diff that will appear after upgrading, and applying a plan will resolve it.
 9"""
10
11import json
12
13from sqlglot import exp
14
15from sqlmesh.core.console import get_console
16
17SQLMESH_DBT_PACKAGE = "sqlmesh.dbt"
18
19
20def migrate_schemas(engine_adapter, schema, **kwargs):  # type: ignore
21    pass
22
23
24def migrate_rows(engine_adapter, schema, **kwargs):  # type: ignore
25    snapshots_table = "_snapshots"
26    if schema:
27        snapshots_table = f"{schema}.{snapshots_table}"
28
29    warning = (
30        "SQLMesh detected that it may not be able to fully migrate the state database. This should not impact "
31        "the migration process, but may result in unexpected changes being reported by the next `sqlmesh plan` "
32        "command. Please run `sqlmesh diff prod` after the migration has completed, before making any new "
33        "changes. If any unexpected changes are reported, consider running a forward-only plan to apply these "
34        "changes and avoid unnecessary backfills: sqlmesh plan prod --forward-only. "
35        "See https://sqlmesh.readthedocs.io/en/stable/concepts/plans/#forward-only-plans for more details.\n"
36    )
37
38    for (snapshot,) in engine_adapter.fetchall(
39        exp.select("snapshot").from_(snapshots_table), quote_identifiers=True
40    ):
41        parsed_snapshot = json.loads(snapshot)
42        node = parsed_snapshot["node"]
43
44        jinja_macros = node.get("jinja_macros") or {}
45        create_builtins_module = jinja_macros.get("create_builtins_module") or ""
46
47        if create_builtins_module == SQLMESH_DBT_PACKAGE:
48            get_console().log_warning(warning)
49            return
SQLMESH_DBT_PACKAGE = 'sqlmesh.dbt'
def migrate_schemas(engine_adapter, schema, **kwargs): 
21def migrate_schemas(engine_adapter, schema, **kwargs):  # type: ignore
22    pass
def migrate_rows(engine_adapter, schema, **kwargs): 
25def migrate_rows(engine_adapter, schema, **kwargs):  # type: ignore
26    snapshots_table = "_snapshots"
27    if schema:
28        snapshots_table = f"{schema}.{snapshots_table}"
29
30    warning = (
31        "SQLMesh detected that it may not be able to fully migrate the state database. This should not impact "
32        "the migration process, but may result in unexpected changes being reported by the next `sqlmesh plan` "
33        "command. Please run `sqlmesh diff prod` after the migration has completed, before making any new "
34        "changes. If any unexpected changes are reported, consider running a forward-only plan to apply these "
35        "changes and avoid unnecessary backfills: sqlmesh plan prod --forward-only. "
36        "See https://sqlmesh.readthedocs.io/en/stable/concepts/plans/#forward-only-plans for more details.\n"
37    )
38
39    for (snapshot,) in engine_adapter.fetchall(
40        exp.select("snapshot").from_(snapshots_table), quote_identifiers=True
41    ):
42        parsed_snapshot = json.loads(snapshot)
43        node = parsed_snapshot["node"]
44
45        jinja_macros = node.get("jinja_macros") or {}
46        create_builtins_module = jinja_macros.get("create_builtins_module") or ""
47
48        if create_builtins_module == SQLMESH_DBT_PACKAGE:
49            get_console().log_warning(warning)
50            return