Edit on GitHub

sqlmesh.core.model.meta

  1from __future__ import annotations
  2
  3import typing as t
  4from enum import Enum
  5from functools import cached_property
  6from typing_extensions import Self
  7
  8from pydantic import Field
  9from sqlglot import Dialect, exp, parse_one
 10from sqlglot.helper import ensure_collection, ensure_list
 11from sqlglot.optimizer.normalize_identifiers import normalize_identifiers
 12
 13from sqlmesh.core import dialect as d
 14from sqlmesh.core.config.common import VirtualEnvironmentMode
 15from sqlmesh.core.constants import LIQUID_CLUSTERING_KEYWORDS
 16from sqlmesh.core.config.linter import LinterConfig
 17from sqlmesh.core.dialect import normalize_model_name
 18from sqlmesh.utils import classproperty
 19from sqlmesh.core.model.common import (
 20    bool_validator,
 21    default_catalog_validator,
 22    depends_on_validator,
 23    properties_validator,
 24    parse_properties,
 25)
 26from sqlmesh.core.model.kind import (
 27    CustomKind,
 28    IncrementalByUniqueKeyKind,
 29    ModelKind,
 30    OnDestructiveChange,
 31    SCDType2ByColumnKind,
 32    SCDType2ByTimeKind,
 33    TimeColumn,
 34    ViewKind,
 35    model_kind_validator,
 36    OnAdditiveChange,
 37)
 38from sqlmesh.core.node import _Node, str_or_exp_to_str
 39from sqlmesh.core.reference import Reference
 40from sqlmesh.utils.date import TimeLike
 41from sqlmesh.utils.errors import ConfigError
 42from sqlmesh.utils.pydantic import (
 43    ValidationInfo,
 44    field_validator,
 45    list_of_fields_validator,
 46    model_validator,
 47    get_dialect,
 48    validation_data,
 49)
 50
 51if t.TYPE_CHECKING:
 52    from sqlmesh.core._typing import CustomMaterializationProperties, SessionProperties
 53    from sqlmesh.core.engine_adapter._typing import GrantsConfig
 54
 55FunctionCall = t.Tuple[str, t.Dict[str, exp.Expr]]
 56
 57
 58class GrantsTargetLayer(str, Enum):
 59    """Target layer(s) where grants should be applied."""
 60
 61    ALL = "all"
 62    PHYSICAL = "physical"
 63    VIRTUAL = "virtual"
 64
 65    @classproperty
 66    def default(cls) -> "GrantsTargetLayer":
 67        return GrantsTargetLayer.VIRTUAL
 68
 69    @property
 70    def is_all(self) -> bool:
 71        return self == GrantsTargetLayer.ALL
 72
 73    @property
 74    def is_physical(self) -> bool:
 75        return self == GrantsTargetLayer.PHYSICAL
 76
 77    @property
 78    def is_virtual(self) -> bool:
 79        return self == GrantsTargetLayer.VIRTUAL
 80
 81    def __str__(self) -> str:
 82        return self.name
 83
 84    def __repr__(self) -> str:
 85        return str(self)
 86
 87
 88class ModelMeta(_Node):
 89    """Metadata for models which can be defined in SQL."""
 90
 91    dialect: str = ""
 92    name: str
 93    kind: ModelKind = ViewKind()
 94    retention: t.Optional[int] = None  # not implemented yet
 95    table_format: t.Optional[str] = None
 96    storage_format: t.Optional[str] = None
 97    partitioned_by_: t.List[exp.Expr] = Field(default=[], alias="partitioned_by")
 98    clustered_by: t.List[exp.Expr] = []
 99    default_catalog: t.Optional[str] = None
100    depends_on_: t.Optional[t.Set[str]] = Field(default=None, alias="depends_on")
101    columns_to_types_: t.Optional[t.Dict[str, exp.DataType]] = Field(default=None, alias="columns")
102    column_descriptions_: t.Optional[t.Dict[str, str]] = Field(
103        default=None, alias="column_descriptions"
104    )
105    audits: t.List[FunctionCall] = []
106    grains: t.List[exp.Expr] = []
107    references: t.List[exp.Expr] = []
108    physical_schema_override: t.Optional[str] = None
109    physical_properties_: t.Optional[exp.Tuple] = Field(default=None, alias="physical_properties")
110    virtual_properties_: t.Optional[exp.Tuple] = Field(default=None, alias="virtual_properties")
111    session_properties_: t.Optional[exp.Tuple] = Field(default=None, alias="session_properties")
112    allow_partials: bool = False
113    signals: t.List[FunctionCall] = []
114    enabled: bool = True
115    physical_version: t.Optional[str] = None
116    gateway: t.Optional[str] = None
117    optimize_query: t.Optional[bool] = None
118    ignored_rules_: t.Optional[t.Set[str]] = Field(
119        default=None, exclude=True, alias="ignored_rules"
120    )
121    formatting: t.Optional[bool] = Field(default=None, exclude=True)
122    virtual_environment_mode: VirtualEnvironmentMode = VirtualEnvironmentMode.default
123    grants_: t.Optional[exp.Tuple] = Field(default=None, alias="grants")
124    grants_target_layer: GrantsTargetLayer = GrantsTargetLayer.default
125
126    _bool_validator = bool_validator
127    _model_kind_validator = model_kind_validator
128    _properties_validator = properties_validator
129    _default_catalog_validator = default_catalog_validator
130    _depends_on_validator = depends_on_validator
131
132    @field_validator("audits", "signals", mode="before")
133    def _func_call_validator(cls, v: t.Any, field: t.Any) -> t.Any:
134        is_signal = getattr(field, "name" if hasattr(field, "name") else "field_name") == "signals"
135
136        return d.extract_function_calls(v, allow_tuples=is_signal)
137
138    @field_validator("tags", mode="before")
139    def _value_or_tuple_validator(cls, v: t.Any, info: ValidationInfo) -> t.Any:
140        return ensure_list(cls._validate_value_or_tuple(v, validation_data(info)))
141
142    @classmethod
143    def _validate_value_or_tuple(
144        cls, v: t.Dict[str, t.Any], data: t.Dict[str, t.Any], normalize: bool = False
145    ) -> t.Any:
146        dialect = data.get("dialect")
147
148        def _normalize(value: t.Any) -> t.Any:
149            return normalize_identifiers(value, dialect=dialect) if normalize else value
150
151        if isinstance(v, exp.Paren):
152            v = [v.unnest()]
153
154        if isinstance(v, (exp.Tuple, exp.Array)):
155            return [_normalize(e).name for e in v.expressions]
156        if isinstance(v, exp.Expr):
157            return _normalize(v).name
158        if isinstance(v, str):
159            value = _normalize(v)
160            return value.name if isinstance(value, exp.Expr) else value
161        if isinstance(v, (list, tuple)):
162            return [cls._validate_value_or_tuple(elm, data, normalize=normalize) for elm in v]
163
164        return v
165
166    @field_validator("table_format", "storage_format", mode="before")
167    def _format_validator(cls, v: t.Any, info: ValidationInfo) -> t.Optional[str]:
168        if isinstance(v, exp.Expr) and not (isinstance(v, (exp.Literal, exp.Identifier))):
169            return v.sql(validation_data(info).get("dialect"))
170        return str_or_exp_to_str(v)
171
172    @field_validator("dialect", mode="before")
173    def _dialect_validator(cls, v: t.Any) -> t.Optional[str]:
174        # dialects are parsed as identifiers and may get normalized as uppercase,
175        # so this ensures they'll be stored as lowercase
176        dialect = str_or_exp_to_str(v)
177        return dialect and dialect.lower()
178
179    @field_validator("physical_version", mode="before")
180    def _physical_version_validator(cls, v: t.Any) -> t.Optional[str]:
181        if v is None:
182            return v
183        return str_or_exp_to_str(v)
184
185    @field_validator("gateway", mode="before")
186    def _gateway_validator(cls, v: t.Any) -> t.Optional[str]:
187        if v is None:
188            return None
189        gateway = str_or_exp_to_str(v)
190        return gateway and gateway.lower()
191
192    @field_validator("partitioned_by_", "clustered_by", mode="before")
193    def _partition_and_cluster_validator(cls, v: t.Any, info: ValidationInfo) -> t.List[exp.Expr]:
194        field = info.field_name or ""
195        dialect = (get_dialect(info) or "").lower()
196
197        if (
198            isinstance(v, list)
199            and all(isinstance(i, str) for i in v)
200            and field == "partitioned_by_"
201        ):
202            # this branch gets hit when we are deserializing from json because `partitioned_by` is stored as a List[str]
203            # however, we should only invoke this if the list contains strings because this validator is also
204            # called by Python models which might pass a List[exp.Expression]
205            string_to_parse = (
206                f"({','.join(v)})"  # recreate the (a, b, c) part of "partitioned_by (a, b, c)"
207            )
208            parsed = parse_one(
209                string_to_parse, into=exp.PartitionedByProperty, dialect=get_dialect(info)
210            )
211            v = parsed.this.expressions if isinstance(parsed.this, exp.Schema) else v
212
213        if isinstance(v, str) and field == "clustered_by":
214            v = [v]
215
216        if isinstance(v, list) and field == "clustered_by" and dialect == "databricks":
217            # When deserializing from JSON, clustered_by is stored as List[str].
218            # Restore keyword sentinels (AUTO/NONE) before list_of_fields_validator normalises
219            # them into quoted columns.
220            v = [
221                exp.Var(this=item.upper())
222                if isinstance(item, str) and item.upper() in LIQUID_CLUSTERING_KEYWORDS
223                else item
224                for item in v
225            ]
226
227        expressions = list_of_fields_validator(v, validation_data(info))
228
229        for expression in expressions:
230            # AUTO and NONE are Databricks liquid clustering keywords, not column references.
231            # Only skip for clustered_by with the Databricks dialect — meaningless elsewhere.
232            if (
233                field == "clustered_by"
234                and dialect == "databricks"
235                and isinstance(expression, exp.Var)
236                and expression.name.upper() in LIQUID_CLUSTERING_KEYWORDS
237            ):
238                continue
239
240            num_cols = len(list(expression.find_all(exp.Column)))
241
242            error_msg: t.Optional[str] = None
243            if num_cols == 0:
244                error_msg = "does not contain a column"
245            elif num_cols > 1:
246                error_msg = "contains multiple columns"
247
248            if error_msg:
249                raise ConfigError(f"Field '{expression}' {error_msg}")
250
251        return expressions
252
253    @field_validator(
254        "columns_to_types_", "derived_columns_to_types", mode="before", check_fields=False
255    )
256    def _columns_validator(
257        cls, v: t.Any, info: ValidationInfo
258    ) -> t.Optional[t.Dict[str, exp.DataType]]:
259        columns_to_types = {}
260        dialect = validation_data(info).get("dialect")
261
262        if isinstance(v, exp.Schema):
263            for column in v.expressions:
264                expr = column.args.get("kind")
265                if not isinstance(expr, exp.DataType):
266                    raise ConfigError(f"Missing data type for column '{column.name}'.")
267
268                expr.meta["dialect"] = dialect
269                columns_to_types[normalize_identifiers(column, dialect=dialect).name] = expr
270
271            return columns_to_types
272
273        if isinstance(v, dict):
274            dialect_obj = Dialect.get_or_raise(dialect)
275            udt = dialect_obj.SUPPORTS_USER_DEFINED_TYPES
276            for k, data_type in v.items():
277                is_string_type = isinstance(data_type, str)
278                expr = exp.DataType.build(data_type, dialect=dialect, udt=udt)
279                # When deserializing from a string (e.g. JSON roundtrip), normalize the type
280                # through the dialect's type system so that aliases (e.g. INT in BigQuery,
281                # which is an alias for INT64/BIGINT) are resolved to their canonical form.
282                # This ensures stable data hash computation across serialization/deserialization
283                # roundtrips. We skip this for DataType objects passed directly (Python API)
284                # since those should be used as-is.
285                if (
286                    is_string_type
287                    and dialect
288                    and expr.this
289                    not in (
290                        exp.DataType.Type.USERDEFINED,
291                        exp.DataType.Type.UNKNOWN,
292                    )
293                ):
294                    sql_repr = expr.sql(dialect=dialect)
295                    try:
296                        normalized = parse_one(sql_repr, read=dialect, into=exp.DataType)
297                        if normalized is not None:
298                            expr = normalized
299                    except Exception:
300                        pass
301                expr.meta["dialect"] = dialect
302                columns_to_types[normalize_identifiers(k, dialect=dialect).name] = expr
303
304            return columns_to_types
305
306        return v
307
308    @field_validator("column_descriptions_", mode="before")
309    def _column_descriptions_validator(
310        cls, vs: t.Any, info: ValidationInfo
311    ) -> t.Optional[t.Dict[str, str]]:
312        data = validation_data(info)
313        dialect = data.get("dialect")
314
315        if vs is None:
316            return None
317
318        if isinstance(vs, exp.Paren):
319            vs = vs.flatten()
320
321        if isinstance(vs, (exp.Tuple, exp.Array)):
322            vs = vs.expressions
323
324        raw_col_descriptions = (
325            vs
326            if isinstance(vs, dict)
327            else {".".join([part.this for part in v.this.parts]): v.expression.name for v in vs}
328        )
329
330        col_descriptions = {
331            normalize_identifiers(k, dialect=dialect).name: v
332            for k, v in raw_col_descriptions.items()
333        }
334
335        columns_to_types = data.get("columns_to_types_")
336        if columns_to_types:
337            from sqlmesh.core.console import get_console
338
339            console = get_console()
340            for column_name in list(col_descriptions):
341                if column_name not in columns_to_types:
342                    console.log_warning(
343                        f"In model '{data.get('name', '<unknown>')}', a description is provided for column '{column_name}' but it is not a column in the model."
344                    )
345                    del col_descriptions[column_name]
346
347        return col_descriptions
348
349    @field_validator("grains", "references", mode="before")
350    def _refs_validator(cls, vs: t.Any, info: ValidationInfo) -> t.List[exp.Expr]:
351        dialect = validation_data(info).get("dialect")
352
353        if isinstance(vs, exp.Paren):
354            vs = vs.unnest()
355
356        if isinstance(vs, (exp.Tuple, exp.Array)):
357            vs = vs.expressions
358        else:
359            vs = [
360                d.parse_one(v, dialect=dialect) if isinstance(v, str) else v
361                for v in ensure_collection(vs)
362            ]
363
364        refs = []
365
366        for v in vs:
367            v = exp.column(v) if isinstance(v, exp.Identifier) else v
368            v.meta["dialect"] = dialect
369            refs.append(v)
370
371        return refs
372
373    @field_validator("ignored_rules_", mode="before")
374    def ignored_rules_validator(cls, vs: t.Any) -> t.Any:
375        return LinterConfig._validate_rules(vs)
376
377    @field_validator("grants_target_layer", mode="before")
378    def _grants_target_layer_validator(cls, v: t.Any) -> t.Any:
379        if isinstance(v, exp.Identifier):
380            return v.this
381        if isinstance(v, exp.Literal) and v.is_string:
382            return v.this
383        return v
384
385    @field_validator("session_properties_", mode="before")
386    def session_properties_validator(cls, v: t.Any, info: ValidationInfo) -> t.Any:
387        # use the generic properties validator to parse the session properties
388        parsed_session_properties = parse_properties(type(cls), v, info)
389        if not parsed_session_properties:
390            return parsed_session_properties
391
392        for eq in parsed_session_properties:
393            prop_name = eq.left.name
394
395            if prop_name == "query_label":
396                query_label = eq.right
397                if not isinstance(
398                    query_label, (exp.Array, exp.Tuple, exp.Paren, d.MacroFunc, d.MacroVar)
399                ):
400                    raise ConfigError(
401                        "Invalid value for `session_properties.query_label`. Must be an array or tuple."
402                    )
403
404                label_tuples: t.List[exp.Expr] = (
405                    [query_label.unnest()]
406                    if isinstance(query_label, exp.Paren)
407                    else query_label.expressions
408                )
409
410                for label_tuple in label_tuples:
411                    if not (
412                        isinstance(label_tuple, exp.Tuple)
413                        and len(label_tuple.expressions) == 2
414                        and all(isinstance(label, exp.Literal) for label in label_tuple.expressions)
415                    ):
416                        raise ConfigError(
417                            "Invalid entry in `session_properties.query_label`. Must be tuples of string literals with length 2."
418                        )
419            elif prop_name == "authorization":
420                authorization = eq.right
421                if not (
422                    isinstance(authorization, exp.Literal) and authorization.is_string
423                ) and not isinstance(authorization, (d.MacroFunc, d.MacroVar)):
424                    raise ConfigError(
425                        "Invalid value for `session_properties.authorization`. Must be a string literal."
426                    )
427            elif prop_name == "query_tags":
428                query_tags = eq.right
429                if isinstance(query_tags, (d.MacroFunc, d.MacroVar)):
430                    continue
431
432                if not isinstance(query_tags, (exp.Map, exp.VarMap)):
433                    raise ConfigError(
434                        "Invalid value for `session_properties.query_tags`. Must be a map."
435                    )
436
437                keys = query_tags.args.get("keys")
438                values = query_tags.args.get("values")
439                if not isinstance(keys, exp.Array) or not isinstance(values, exp.Array):
440                    raise ConfigError(
441                        "Invalid value for `session_properties.query_tags`. Must be a map with array "
442                        "keys and array values."
443                    )
444
445                for key, value in zip(keys.expressions, values.expressions):
446                    if not isinstance(key, exp.Literal) or not key.is_string:
447                        raise ConfigError(
448                            "Invalid key in `session_properties.query_tags`. Keys must be string literals."
449                        )
450
451                    if not (
452                        isinstance(value, exp.Null)
453                        or (isinstance(value, exp.Literal) and value.is_string)
454                    ):
455                        raise ConfigError(
456                            "Invalid value in `session_properties.query_tags`. Values must be string "
457                            "literals or NULL."
458                        )
459
460        return parsed_session_properties
461
462    @model_validator(mode="before")
463    def _pre_root_validator(cls, data: t.Any) -> t.Any:
464        if not isinstance(data, dict):
465            return data
466
467        grain = data.pop("grain", None)
468        if grain:
469            grains = data.get("grains")
470            if grains:
471                raise ConfigError(
472                    f"Cannot use argument 'grain' ({grain}) with 'grains' ({grains}), use only grains"
473                )
474            data["grains"] = ensure_list(grain)
475
476        table_properties = data.pop("table_properties", None)
477        if table_properties:
478            if not isinstance(table_properties, str):
479                # Do not warn when deserializing from the state.
480                model_name = data["name"]
481                from sqlmesh.core.console import get_console
482
483                get_console().log_warning(
484                    f"Model '{model_name}' is using the `table_properties` attribute which is deprecated. Please use `physical_properties` instead."
485                )
486            physical_properties = data.get("physical_properties")
487            if physical_properties:
488                raise ConfigError(
489                    f"Cannot use argument 'table_properties' ({table_properties}) with 'physical_properties' ({physical_properties}), use only physical_properties."
490                )
491
492            data["physical_properties"] = table_properties
493
494        return data
495
496    @model_validator(mode="after")
497    def _root_validator(self) -> Self:
498        kind: t.Any = self.kind
499
500        for field in ("partitioned_by_", "clustered_by"):
501            if (
502                getattr(self, field, None)
503                and not kind.is_materialized
504                and not (kind.is_view and kind.materialized)
505            ):
506                name = field[:-1] if field.endswith("_") else field
507                raise ValueError(f"{name} field cannot be set for {kind.name} models")
508        if kind.is_incremental_by_partition and not getattr(self, "partitioned_by_", None):
509            raise ValueError(f"partitioned_by field is required for {kind.name} models")
510
511        # needs to be in a mode=after model validator so that the field validators have run to convert from Expression -> str
512        if (storage_format := self.storage_format) and storage_format.lower() in {
513            "iceberg",
514            "hive",
515            "hudi",
516            "delta",
517        }:
518            from sqlmesh.core.console import get_console
519
520            get_console().log_warning(
521                f"Model {self.name} has `storage_format` set to a table format '{storage_format}' which is deprecated. Please use the `table_format` property instead."
522            )
523
524        # Validate grants configuration for model kind support
525        if self.grants is not None and not kind.supports_grants:
526            raise ValueError(f"grants cannot be set for {kind.name} models")
527
528        return self
529
530    @property
531    def time_column(self) -> t.Optional[TimeColumn]:
532        """The time column for incremental models."""
533        return getattr(self.kind, "time_column", None)
534
535    @property
536    def unique_key(self) -> t.List[exp.Expr]:
537        if isinstance(
538            self.kind, (SCDType2ByTimeKind, SCDType2ByColumnKind, IncrementalByUniqueKeyKind)
539        ):
540            return self.kind.unique_key
541        return []
542
543    @property
544    def column_descriptions(self) -> t.Dict[str, str]:
545        """A dictionary of column names to annotation comments."""
546        return self.column_descriptions_ or {}
547
548    @property
549    def lookback(self) -> int:
550        """The incremental lookback window."""
551        return getattr(self.kind, "lookback", 0) or 0
552
553    def lookback_start(self, start: TimeLike) -> TimeLike:
554        if self.lookback == 0:
555            return start
556
557        for _ in range(self.lookback):
558            start = self.interval_unit.cron_prev(start)
559        return start
560
561    @property
562    def batch_size(self) -> t.Optional[int]:
563        """The maximal number of units in a single task for a backfill."""
564        return getattr(self.kind, "batch_size", None)
565
566    @property
567    def batch_concurrency(self) -> t.Optional[int]:
568        """The maximal number of batches that can run concurrently for a backfill."""
569        return getattr(self.kind, "batch_concurrency", None)
570
571    @cached_property
572    def physical_properties(self) -> t.Dict[str, exp.Expr]:
573        """A dictionary of properties that will be applied to the physical layer. It replaces table_properties which is deprecated."""
574        if self.physical_properties_:
575            return {e.this.name: e.expression for e in self.physical_properties_.expressions}
576        return {}
577
578    @cached_property
579    def virtual_properties(self) -> t.Dict[str, exp.Expr]:
580        """A dictionary of properties that will be applied to the virtual layer."""
581        if self.virtual_properties_:
582            return {e.this.name: e.expression for e in self.virtual_properties_.expressions}
583        return {}
584
585    @property
586    def session_properties(self) -> SessionProperties:
587        """A dictionary of session properties."""
588        if not self.session_properties_:
589            return {}
590
591        return d.interpret_key_value_pairs(self.session_properties_)
592
593    @property
594    def custom_materialization_properties(self) -> CustomMaterializationProperties:
595        if isinstance(self.kind, CustomKind):
596            return self.kind.materialization_properties
597        return {}
598
599    @cached_property
600    def grants(self) -> t.Optional[GrantsConfig]:
601        """A dictionary of grants mapping permission names to lists of grantees."""
602
603        if self.grants_ is None:
604            return None
605
606        if not self.grants_.expressions:
607            return {}
608
609        grants_dict = {}
610        for eq_expr in self.grants_.expressions:
611            try:
612                permission_name = self._validate_config_expression(eq_expr.left)
613                grantee_list = self._validate_nested_config_values(eq_expr.expression)
614                grants_dict[permission_name] = grantee_list
615            except ConfigError as e:
616                permission_name = (
617                    eq_expr.left.name if hasattr(eq_expr.left, "name") else str(eq_expr.left)
618                )
619                raise ConfigError(f"Invalid grants configuration for '{permission_name}': {e}")
620
621        return grants_dict if grants_dict else None
622
623    @property
624    def all_references(self) -> t.List[Reference]:
625        """All references including grains."""
626        return [Reference(model_name=self.name, expression=e, unique=True) for e in self.grains] + [
627            Reference(model_name=self.name, expression=e, unique=True) for e in self.references
628        ]
629
630    @property
631    def on(self) -> t.List[str]:
632        """The grains to be used as join condition in table_diff."""
633
634        on: t.List[str] = []
635        for expr in [ref.expression for ref in self.all_references if ref.unique]:
636            if isinstance(expr, exp.Tuple):
637                on.extend([key.this.sql(dialect=self.dialect) for key in expr.expressions])
638            else:
639                # Handle a single Column or Paren expression
640                on.append(expr.this.sql(dialect=self.dialect))
641
642        return on
643
644    @property
645    def managed_columns(self) -> t.Dict[str, exp.DataType]:
646        return getattr(self.kind, "managed_columns", {})
647
648    @property
649    def when_matched(self) -> t.Optional[exp.Whens]:
650        if isinstance(self.kind, IncrementalByUniqueKeyKind):
651            return self.kind.when_matched
652        return None
653
654    @property
655    def merge_filter(self) -> t.Optional[exp.Expr]:
656        if isinstance(self.kind, IncrementalByUniqueKeyKind):
657            return self.kind.merge_filter
658        return None
659
660    @property
661    def catalog(self) -> t.Optional[str]:
662        """Returns the catalog of a model."""
663        return self.fully_qualified_table.catalog
664
665    @cached_property
666    def fully_qualified_table(self) -> exp.Table:
667        return exp.to_table(self.fqn)
668
669    @cached_property
670    def fqn(self) -> str:
671        return normalize_model_name(
672            self.name, default_catalog=self.default_catalog, dialect=self.dialect
673        )
674
675    @property
676    def on_destructive_change(self) -> OnDestructiveChange:
677        return getattr(self.kind, "on_destructive_change", OnDestructiveChange.ALLOW)
678
679    @property
680    def on_additive_change(self) -> OnAdditiveChange:
681        """Return the model's additive change setting if it has one."""
682        return getattr(self.kind, "on_additive_change", OnAdditiveChange.ALLOW)
683
684    @property
685    def ignored_rules(self) -> t.Set[str]:
686        return self.ignored_rules_ or set()
687
688    def _validate_config_expression(self, expr: exp.Expr) -> str:
689        if isinstance(expr, (d.MacroFunc, d.MacroVar)):
690            raise ConfigError(f"Unresolved macro: {expr.sql(dialect=self.dialect)}")
691
692        if isinstance(expr, exp.Null):
693            raise ConfigError("NULL value")
694
695        if isinstance(expr, exp.Literal):
696            return str(expr.this).strip()
697        if isinstance(expr, (exp.Column, exp.Identifier)):
698            return expr.name
699        return expr.sql(dialect=self.dialect).strip()
700
701    def _validate_nested_config_values(self, value_expr: exp.Expr) -> t.List[str]:
702        result = []
703
704        def flatten_expr(expr: exp.Expr) -> None:
705            if isinstance(expr, exp.Array):
706                for elem in expr.expressions:
707                    flatten_expr(elem)
708            elif isinstance(expr, (exp.Tuple, exp.Paren)):
709                expressions = [expr.unnest()] if isinstance(expr, exp.Paren) else expr.expressions
710                for elem in expressions:
711                    flatten_expr(elem)
712            else:
713                result.append(self._validate_config_expression(expr))
714
715        flatten_expr(value_expr)
716        return result
FunctionCall = typing.Tuple[str, typing.Dict[str, sqlglot.expressions.core.Expr]]
class GrantsTargetLayer(builtins.str, enum.Enum):
59class GrantsTargetLayer(str, Enum):
60    """Target layer(s) where grants should be applied."""
61
62    ALL = "all"
63    PHYSICAL = "physical"
64    VIRTUAL = "virtual"
65
66    @classproperty
67    def default(cls) -> "GrantsTargetLayer":
68        return GrantsTargetLayer.VIRTUAL
69
70    @property
71    def is_all(self) -> bool:
72        return self == GrantsTargetLayer.ALL
73
74    @property
75    def is_physical(self) -> bool:
76        return self == GrantsTargetLayer.PHYSICAL
77
78    @property
79    def is_virtual(self) -> bool:
80        return self == GrantsTargetLayer.VIRTUAL
81
82    def __str__(self) -> str:
83        return self.name
84
85    def __repr__(self) -> str:
86        return str(self)

Target layer(s) where grants should be applied.

ALL = ALL
PHYSICAL = PHYSICAL
VIRTUAL = VIRTUAL
default: GrantsTargetLayer
66    @classproperty
67    def default(cls) -> "GrantsTargetLayer":
68        return GrantsTargetLayer.VIRTUAL

Target layer(s) where grants should be applied.

is_all: bool
70    @property
71    def is_all(self) -> bool:
72        return self == GrantsTargetLayer.ALL
is_physical: bool
74    @property
75    def is_physical(self) -> bool:
76        return self == GrantsTargetLayer.PHYSICAL
is_virtual: bool
78    @property
79    def is_virtual(self) -> bool:
80        return self == GrantsTargetLayer.VIRTUAL
Inherited Members
enum.Enum
name
value
builtins.str
encode
replace
split
rsplit
join
capitalize
casefold
title
center
count
expandtabs
find
partition
index
ljust
lower
lstrip
rfind
rindex
rjust
rstrip
rpartition
splitlines
strip
swapcase
translate
upper
startswith
endswith
removeprefix
removesuffix
isascii
islower
isupper
istitle
isspace
isdecimal
isdigit
isnumeric
isalpha
isalnum
isidentifier
isprintable
zfill
format
format_map
maketrans
class ModelMeta(sqlmesh.core.node._Node):
 89class ModelMeta(_Node):
 90    """Metadata for models which can be defined in SQL."""
 91
 92    dialect: str = ""
 93    name: str
 94    kind: ModelKind = ViewKind()
 95    retention: t.Optional[int] = None  # not implemented yet
 96    table_format: t.Optional[str] = None
 97    storage_format: t.Optional[str] = None
 98    partitioned_by_: t.List[exp.Expr] = Field(default=[], alias="partitioned_by")
 99    clustered_by: t.List[exp.Expr] = []
100    default_catalog: t.Optional[str] = None
101    depends_on_: t.Optional[t.Set[str]] = Field(default=None, alias="depends_on")
102    columns_to_types_: t.Optional[t.Dict[str, exp.DataType]] = Field(default=None, alias="columns")
103    column_descriptions_: t.Optional[t.Dict[str, str]] = Field(
104        default=None, alias="column_descriptions"
105    )
106    audits: t.List[FunctionCall] = []
107    grains: t.List[exp.Expr] = []
108    references: t.List[exp.Expr] = []
109    physical_schema_override: t.Optional[str] = None
110    physical_properties_: t.Optional[exp.Tuple] = Field(default=None, alias="physical_properties")
111    virtual_properties_: t.Optional[exp.Tuple] = Field(default=None, alias="virtual_properties")
112    session_properties_: t.Optional[exp.Tuple] = Field(default=None, alias="session_properties")
113    allow_partials: bool = False
114    signals: t.List[FunctionCall] = []
115    enabled: bool = True
116    physical_version: t.Optional[str] = None
117    gateway: t.Optional[str] = None
118    optimize_query: t.Optional[bool] = None
119    ignored_rules_: t.Optional[t.Set[str]] = Field(
120        default=None, exclude=True, alias="ignored_rules"
121    )
122    formatting: t.Optional[bool] = Field(default=None, exclude=True)
123    virtual_environment_mode: VirtualEnvironmentMode = VirtualEnvironmentMode.default
124    grants_: t.Optional[exp.Tuple] = Field(default=None, alias="grants")
125    grants_target_layer: GrantsTargetLayer = GrantsTargetLayer.default
126
127    _bool_validator = bool_validator
128    _model_kind_validator = model_kind_validator
129    _properties_validator = properties_validator
130    _default_catalog_validator = default_catalog_validator
131    _depends_on_validator = depends_on_validator
132
133    @field_validator("audits", "signals", mode="before")
134    def _func_call_validator(cls, v: t.Any, field: t.Any) -> t.Any:
135        is_signal = getattr(field, "name" if hasattr(field, "name") else "field_name") == "signals"
136
137        return d.extract_function_calls(v, allow_tuples=is_signal)
138
139    @field_validator("tags", mode="before")
140    def _value_or_tuple_validator(cls, v: t.Any, info: ValidationInfo) -> t.Any:
141        return ensure_list(cls._validate_value_or_tuple(v, validation_data(info)))
142
143    @classmethod
144    def _validate_value_or_tuple(
145        cls, v: t.Dict[str, t.Any], data: t.Dict[str, t.Any], normalize: bool = False
146    ) -> t.Any:
147        dialect = data.get("dialect")
148
149        def _normalize(value: t.Any) -> t.Any:
150            return normalize_identifiers(value, dialect=dialect) if normalize else value
151
152        if isinstance(v, exp.Paren):
153            v = [v.unnest()]
154
155        if isinstance(v, (exp.Tuple, exp.Array)):
156            return [_normalize(e).name for e in v.expressions]
157        if isinstance(v, exp.Expr):
158            return _normalize(v).name
159        if isinstance(v, str):
160            value = _normalize(v)
161            return value.name if isinstance(value, exp.Expr) else value
162        if isinstance(v, (list, tuple)):
163            return [cls._validate_value_or_tuple(elm, data, normalize=normalize) for elm in v]
164
165        return v
166
167    @field_validator("table_format", "storage_format", mode="before")
168    def _format_validator(cls, v: t.Any, info: ValidationInfo) -> t.Optional[str]:
169        if isinstance(v, exp.Expr) and not (isinstance(v, (exp.Literal, exp.Identifier))):
170            return v.sql(validation_data(info).get("dialect"))
171        return str_or_exp_to_str(v)
172
173    @field_validator("dialect", mode="before")
174    def _dialect_validator(cls, v: t.Any) -> t.Optional[str]:
175        # dialects are parsed as identifiers and may get normalized as uppercase,
176        # so this ensures they'll be stored as lowercase
177        dialect = str_or_exp_to_str(v)
178        return dialect and dialect.lower()
179
180    @field_validator("physical_version", mode="before")
181    def _physical_version_validator(cls, v: t.Any) -> t.Optional[str]:
182        if v is None:
183            return v
184        return str_or_exp_to_str(v)
185
186    @field_validator("gateway", mode="before")
187    def _gateway_validator(cls, v: t.Any) -> t.Optional[str]:
188        if v is None:
189            return None
190        gateway = str_or_exp_to_str(v)
191        return gateway and gateway.lower()
192
193    @field_validator("partitioned_by_", "clustered_by", mode="before")
194    def _partition_and_cluster_validator(cls, v: t.Any, info: ValidationInfo) -> t.List[exp.Expr]:
195        field = info.field_name or ""
196        dialect = (get_dialect(info) or "").lower()
197
198        if (
199            isinstance(v, list)
200            and all(isinstance(i, str) for i in v)
201            and field == "partitioned_by_"
202        ):
203            # this branch gets hit when we are deserializing from json because `partitioned_by` is stored as a List[str]
204            # however, we should only invoke this if the list contains strings because this validator is also
205            # called by Python models which might pass a List[exp.Expression]
206            string_to_parse = (
207                f"({','.join(v)})"  # recreate the (a, b, c) part of "partitioned_by (a, b, c)"
208            )
209            parsed = parse_one(
210                string_to_parse, into=exp.PartitionedByProperty, dialect=get_dialect(info)
211            )
212            v = parsed.this.expressions if isinstance(parsed.this, exp.Schema) else v
213
214        if isinstance(v, str) and field == "clustered_by":
215            v = [v]
216
217        if isinstance(v, list) and field == "clustered_by" and dialect == "databricks":
218            # When deserializing from JSON, clustered_by is stored as List[str].
219            # Restore keyword sentinels (AUTO/NONE) before list_of_fields_validator normalises
220            # them into quoted columns.
221            v = [
222                exp.Var(this=item.upper())
223                if isinstance(item, str) and item.upper() in LIQUID_CLUSTERING_KEYWORDS
224                else item
225                for item in v
226            ]
227
228        expressions = list_of_fields_validator(v, validation_data(info))
229
230        for expression in expressions:
231            # AUTO and NONE are Databricks liquid clustering keywords, not column references.
232            # Only skip for clustered_by with the Databricks dialect — meaningless elsewhere.
233            if (
234                field == "clustered_by"
235                and dialect == "databricks"
236                and isinstance(expression, exp.Var)
237                and expression.name.upper() in LIQUID_CLUSTERING_KEYWORDS
238            ):
239                continue
240
241            num_cols = len(list(expression.find_all(exp.Column)))
242
243            error_msg: t.Optional[str] = None
244            if num_cols == 0:
245                error_msg = "does not contain a column"
246            elif num_cols > 1:
247                error_msg = "contains multiple columns"
248
249            if error_msg:
250                raise ConfigError(f"Field '{expression}' {error_msg}")
251
252        return expressions
253
254    @field_validator(
255        "columns_to_types_", "derived_columns_to_types", mode="before", check_fields=False
256    )
257    def _columns_validator(
258        cls, v: t.Any, info: ValidationInfo
259    ) -> t.Optional[t.Dict[str, exp.DataType]]:
260        columns_to_types = {}
261        dialect = validation_data(info).get("dialect")
262
263        if isinstance(v, exp.Schema):
264            for column in v.expressions:
265                expr = column.args.get("kind")
266                if not isinstance(expr, exp.DataType):
267                    raise ConfigError(f"Missing data type for column '{column.name}'.")
268
269                expr.meta["dialect"] = dialect
270                columns_to_types[normalize_identifiers(column, dialect=dialect).name] = expr
271
272            return columns_to_types
273
274        if isinstance(v, dict):
275            dialect_obj = Dialect.get_or_raise(dialect)
276            udt = dialect_obj.SUPPORTS_USER_DEFINED_TYPES
277            for k, data_type in v.items():
278                is_string_type = isinstance(data_type, str)
279                expr = exp.DataType.build(data_type, dialect=dialect, udt=udt)
280                # When deserializing from a string (e.g. JSON roundtrip), normalize the type
281                # through the dialect's type system so that aliases (e.g. INT in BigQuery,
282                # which is an alias for INT64/BIGINT) are resolved to their canonical form.
283                # This ensures stable data hash computation across serialization/deserialization
284                # roundtrips. We skip this for DataType objects passed directly (Python API)
285                # since those should be used as-is.
286                if (
287                    is_string_type
288                    and dialect
289                    and expr.this
290                    not in (
291                        exp.DataType.Type.USERDEFINED,
292                        exp.DataType.Type.UNKNOWN,
293                    )
294                ):
295                    sql_repr = expr.sql(dialect=dialect)
296                    try:
297                        normalized = parse_one(sql_repr, read=dialect, into=exp.DataType)
298                        if normalized is not None:
299                            expr = normalized
300                    except Exception:
301                        pass
302                expr.meta["dialect"] = dialect
303                columns_to_types[normalize_identifiers(k, dialect=dialect).name] = expr
304
305            return columns_to_types
306
307        return v
308
309    @field_validator("column_descriptions_", mode="before")
310    def _column_descriptions_validator(
311        cls, vs: t.Any, info: ValidationInfo
312    ) -> t.Optional[t.Dict[str, str]]:
313        data = validation_data(info)
314        dialect = data.get("dialect")
315
316        if vs is None:
317            return None
318
319        if isinstance(vs, exp.Paren):
320            vs = vs.flatten()
321
322        if isinstance(vs, (exp.Tuple, exp.Array)):
323            vs = vs.expressions
324
325        raw_col_descriptions = (
326            vs
327            if isinstance(vs, dict)
328            else {".".join([part.this for part in v.this.parts]): v.expression.name for v in vs}
329        )
330
331        col_descriptions = {
332            normalize_identifiers(k, dialect=dialect).name: v
333            for k, v in raw_col_descriptions.items()
334        }
335
336        columns_to_types = data.get("columns_to_types_")
337        if columns_to_types:
338            from sqlmesh.core.console import get_console
339
340            console = get_console()
341            for column_name in list(col_descriptions):
342                if column_name not in columns_to_types:
343                    console.log_warning(
344                        f"In model '{data.get('name', '<unknown>')}', a description is provided for column '{column_name}' but it is not a column in the model."
345                    )
346                    del col_descriptions[column_name]
347
348        return col_descriptions
349
350    @field_validator("grains", "references", mode="before")
351    def _refs_validator(cls, vs: t.Any, info: ValidationInfo) -> t.List[exp.Expr]:
352        dialect = validation_data(info).get("dialect")
353
354        if isinstance(vs, exp.Paren):
355            vs = vs.unnest()
356
357        if isinstance(vs, (exp.Tuple, exp.Array)):
358            vs = vs.expressions
359        else:
360            vs = [
361                d.parse_one(v, dialect=dialect) if isinstance(v, str) else v
362                for v in ensure_collection(vs)
363            ]
364
365        refs = []
366
367        for v in vs:
368            v = exp.column(v) if isinstance(v, exp.Identifier) else v
369            v.meta["dialect"] = dialect
370            refs.append(v)
371
372        return refs
373
374    @field_validator("ignored_rules_", mode="before")
375    def ignored_rules_validator(cls, vs: t.Any) -> t.Any:
376        return LinterConfig._validate_rules(vs)
377
378    @field_validator("grants_target_layer", mode="before")
379    def _grants_target_layer_validator(cls, v: t.Any) -> t.Any:
380        if isinstance(v, exp.Identifier):
381            return v.this
382        if isinstance(v, exp.Literal) and v.is_string:
383            return v.this
384        return v
385
386    @field_validator("session_properties_", mode="before")
387    def session_properties_validator(cls, v: t.Any, info: ValidationInfo) -> t.Any:
388        # use the generic properties validator to parse the session properties
389        parsed_session_properties = parse_properties(type(cls), v, info)
390        if not parsed_session_properties:
391            return parsed_session_properties
392
393        for eq in parsed_session_properties:
394            prop_name = eq.left.name
395
396            if prop_name == "query_label":
397                query_label = eq.right
398                if not isinstance(
399                    query_label, (exp.Array, exp.Tuple, exp.Paren, d.MacroFunc, d.MacroVar)
400                ):
401                    raise ConfigError(
402                        "Invalid value for `session_properties.query_label`. Must be an array or tuple."
403                    )
404
405                label_tuples: t.List[exp.Expr] = (
406                    [query_label.unnest()]
407                    if isinstance(query_label, exp.Paren)
408                    else query_label.expressions
409                )
410
411                for label_tuple in label_tuples:
412                    if not (
413                        isinstance(label_tuple, exp.Tuple)
414                        and len(label_tuple.expressions) == 2
415                        and all(isinstance(label, exp.Literal) for label in label_tuple.expressions)
416                    ):
417                        raise ConfigError(
418                            "Invalid entry in `session_properties.query_label`. Must be tuples of string literals with length 2."
419                        )
420            elif prop_name == "authorization":
421                authorization = eq.right
422                if not (
423                    isinstance(authorization, exp.Literal) and authorization.is_string
424                ) and not isinstance(authorization, (d.MacroFunc, d.MacroVar)):
425                    raise ConfigError(
426                        "Invalid value for `session_properties.authorization`. Must be a string literal."
427                    )
428            elif prop_name == "query_tags":
429                query_tags = eq.right
430                if isinstance(query_tags, (d.MacroFunc, d.MacroVar)):
431                    continue
432
433                if not isinstance(query_tags, (exp.Map, exp.VarMap)):
434                    raise ConfigError(
435                        "Invalid value for `session_properties.query_tags`. Must be a map."
436                    )
437
438                keys = query_tags.args.get("keys")
439                values = query_tags.args.get("values")
440                if not isinstance(keys, exp.Array) or not isinstance(values, exp.Array):
441                    raise ConfigError(
442                        "Invalid value for `session_properties.query_tags`. Must be a map with array "
443                        "keys and array values."
444                    )
445
446                for key, value in zip(keys.expressions, values.expressions):
447                    if not isinstance(key, exp.Literal) or not key.is_string:
448                        raise ConfigError(
449                            "Invalid key in `session_properties.query_tags`. Keys must be string literals."
450                        )
451
452                    if not (
453                        isinstance(value, exp.Null)
454                        or (isinstance(value, exp.Literal) and value.is_string)
455                    ):
456                        raise ConfigError(
457                            "Invalid value in `session_properties.query_tags`. Values must be string "
458                            "literals or NULL."
459                        )
460
461        return parsed_session_properties
462
463    @model_validator(mode="before")
464    def _pre_root_validator(cls, data: t.Any) -> t.Any:
465        if not isinstance(data, dict):
466            return data
467
468        grain = data.pop("grain", None)
469        if grain:
470            grains = data.get("grains")
471            if grains:
472                raise ConfigError(
473                    f"Cannot use argument 'grain' ({grain}) with 'grains' ({grains}), use only grains"
474                )
475            data["grains"] = ensure_list(grain)
476
477        table_properties = data.pop("table_properties", None)
478        if table_properties:
479            if not isinstance(table_properties, str):
480                # Do not warn when deserializing from the state.
481                model_name = data["name"]
482                from sqlmesh.core.console import get_console
483
484                get_console().log_warning(
485                    f"Model '{model_name}' is using the `table_properties` attribute which is deprecated. Please use `physical_properties` instead."
486                )
487            physical_properties = data.get("physical_properties")
488            if physical_properties:
489                raise ConfigError(
490                    f"Cannot use argument 'table_properties' ({table_properties}) with 'physical_properties' ({physical_properties}), use only physical_properties."
491                )
492
493            data["physical_properties"] = table_properties
494
495        return data
496
497    @model_validator(mode="after")
498    def _root_validator(self) -> Self:
499        kind: t.Any = self.kind
500
501        for field in ("partitioned_by_", "clustered_by"):
502            if (
503                getattr(self, field, None)
504                and not kind.is_materialized
505                and not (kind.is_view and kind.materialized)
506            ):
507                name = field[:-1] if field.endswith("_") else field
508                raise ValueError(f"{name} field cannot be set for {kind.name} models")
509        if kind.is_incremental_by_partition and not getattr(self, "partitioned_by_", None):
510            raise ValueError(f"partitioned_by field is required for {kind.name} models")
511
512        # needs to be in a mode=after model validator so that the field validators have run to convert from Expression -> str
513        if (storage_format := self.storage_format) and storage_format.lower() in {
514            "iceberg",
515            "hive",
516            "hudi",
517            "delta",
518        }:
519            from sqlmesh.core.console import get_console
520
521            get_console().log_warning(
522                f"Model {self.name} has `storage_format` set to a table format '{storage_format}' which is deprecated. Please use the `table_format` property instead."
523            )
524
525        # Validate grants configuration for model kind support
526        if self.grants is not None and not kind.supports_grants:
527            raise ValueError(f"grants cannot be set for {kind.name} models")
528
529        return self
530
531    @property
532    def time_column(self) -> t.Optional[TimeColumn]:
533        """The time column for incremental models."""
534        return getattr(self.kind, "time_column", None)
535
536    @property
537    def unique_key(self) -> t.List[exp.Expr]:
538        if isinstance(
539            self.kind, (SCDType2ByTimeKind, SCDType2ByColumnKind, IncrementalByUniqueKeyKind)
540        ):
541            return self.kind.unique_key
542        return []
543
544    @property
545    def column_descriptions(self) -> t.Dict[str, str]:
546        """A dictionary of column names to annotation comments."""
547        return self.column_descriptions_ or {}
548
549    @property
550    def lookback(self) -> int:
551        """The incremental lookback window."""
552        return getattr(self.kind, "lookback", 0) or 0
553
554    def lookback_start(self, start: TimeLike) -> TimeLike:
555        if self.lookback == 0:
556            return start
557
558        for _ in range(self.lookback):
559            start = self.interval_unit.cron_prev(start)
560        return start
561
562    @property
563    def batch_size(self) -> t.Optional[int]:
564        """The maximal number of units in a single task for a backfill."""
565        return getattr(self.kind, "batch_size", None)
566
567    @property
568    def batch_concurrency(self) -> t.Optional[int]:
569        """The maximal number of batches that can run concurrently for a backfill."""
570        return getattr(self.kind, "batch_concurrency", None)
571
572    @cached_property
573    def physical_properties(self) -> t.Dict[str, exp.Expr]:
574        """A dictionary of properties that will be applied to the physical layer. It replaces table_properties which is deprecated."""
575        if self.physical_properties_:
576            return {e.this.name: e.expression for e in self.physical_properties_.expressions}
577        return {}
578
579    @cached_property
580    def virtual_properties(self) -> t.Dict[str, exp.Expr]:
581        """A dictionary of properties that will be applied to the virtual layer."""
582        if self.virtual_properties_:
583            return {e.this.name: e.expression for e in self.virtual_properties_.expressions}
584        return {}
585
586    @property
587    def session_properties(self) -> SessionProperties:
588        """A dictionary of session properties."""
589        if not self.session_properties_:
590            return {}
591
592        return d.interpret_key_value_pairs(self.session_properties_)
593
594    @property
595    def custom_materialization_properties(self) -> CustomMaterializationProperties:
596        if isinstance(self.kind, CustomKind):
597            return self.kind.materialization_properties
598        return {}
599
600    @cached_property
601    def grants(self) -> t.Optional[GrantsConfig]:
602        """A dictionary of grants mapping permission names to lists of grantees."""
603
604        if self.grants_ is None:
605            return None
606
607        if not self.grants_.expressions:
608            return {}
609
610        grants_dict = {}
611        for eq_expr in self.grants_.expressions:
612            try:
613                permission_name = self._validate_config_expression(eq_expr.left)
614                grantee_list = self._validate_nested_config_values(eq_expr.expression)
615                grants_dict[permission_name] = grantee_list
616            except ConfigError as e:
617                permission_name = (
618                    eq_expr.left.name if hasattr(eq_expr.left, "name") else str(eq_expr.left)
619                )
620                raise ConfigError(f"Invalid grants configuration for '{permission_name}': {e}")
621
622        return grants_dict if grants_dict else None
623
624    @property
625    def all_references(self) -> t.List[Reference]:
626        """All references including grains."""
627        return [Reference(model_name=self.name, expression=e, unique=True) for e in self.grains] + [
628            Reference(model_name=self.name, expression=e, unique=True) for e in self.references
629        ]
630
631    @property
632    def on(self) -> t.List[str]:
633        """The grains to be used as join condition in table_diff."""
634
635        on: t.List[str] = []
636        for expr in [ref.expression for ref in self.all_references if ref.unique]:
637            if isinstance(expr, exp.Tuple):
638                on.extend([key.this.sql(dialect=self.dialect) for key in expr.expressions])
639            else:
640                # Handle a single Column or Paren expression
641                on.append(expr.this.sql(dialect=self.dialect))
642
643        return on
644
645    @property
646    def managed_columns(self) -> t.Dict[str, exp.DataType]:
647        return getattr(self.kind, "managed_columns", {})
648
649    @property
650    def when_matched(self) -> t.Optional[exp.Whens]:
651        if isinstance(self.kind, IncrementalByUniqueKeyKind):
652            return self.kind.when_matched
653        return None
654
655    @property
656    def merge_filter(self) -> t.Optional[exp.Expr]:
657        if isinstance(self.kind, IncrementalByUniqueKeyKind):
658            return self.kind.merge_filter
659        return None
660
661    @property
662    def catalog(self) -> t.Optional[str]:
663        """Returns the catalog of a model."""
664        return self.fully_qualified_table.catalog
665
666    @cached_property
667    def fully_qualified_table(self) -> exp.Table:
668        return exp.to_table(self.fqn)
669
670    @cached_property
671    def fqn(self) -> str:
672        return normalize_model_name(
673            self.name, default_catalog=self.default_catalog, dialect=self.dialect
674        )
675
676    @property
677    def on_destructive_change(self) -> OnDestructiveChange:
678        return getattr(self.kind, "on_destructive_change", OnDestructiveChange.ALLOW)
679
680    @property
681    def on_additive_change(self) -> OnAdditiveChange:
682        """Return the model's additive change setting if it has one."""
683        return getattr(self.kind, "on_additive_change", OnAdditiveChange.ALLOW)
684
685    @property
686    def ignored_rules(self) -> t.Set[str]:
687        return self.ignored_rules_ or set()
688
689    def _validate_config_expression(self, expr: exp.Expr) -> str:
690        if isinstance(expr, (d.MacroFunc, d.MacroVar)):
691            raise ConfigError(f"Unresolved macro: {expr.sql(dialect=self.dialect)}")
692
693        if isinstance(expr, exp.Null):
694            raise ConfigError("NULL value")
695
696        if isinstance(expr, exp.Literal):
697            return str(expr.this).strip()
698        if isinstance(expr, (exp.Column, exp.Identifier)):
699            return expr.name
700        return expr.sql(dialect=self.dialect).strip()
701
702    def _validate_nested_config_values(self, value_expr: exp.Expr) -> t.List[str]:
703        result = []
704
705        def flatten_expr(expr: exp.Expr) -> None:
706            if isinstance(expr, exp.Array):
707                for elem in expr.expressions:
708                    flatten_expr(elem)
709            elif isinstance(expr, (exp.Tuple, exp.Paren)):
710                expressions = [expr.unnest()] if isinstance(expr, exp.Paren) else expr.expressions
711                for elem in expressions:
712                    flatten_expr(elem)
713            else:
714                result.append(self._validate_config_expression(expr))
715
716        flatten_expr(value_expr)
717        return result

Metadata for models which can be defined in SQL.

dialect: str
name: str
retention: Optional[int]
table_format: Optional[str]
storage_format: Optional[str]
partitioned_by_: List[sqlglot.expressions.core.Expr]
clustered_by: List[sqlglot.expressions.core.Expr]
default_catalog: Optional[str]
depends_on_: Optional[Set[str]]
columns_to_types_: Optional[Dict[str, sqlglot.expressions.datatypes.DataType]]
column_descriptions_: Optional[Dict[str, str]]
audits: List[Tuple[str, Dict[str, sqlglot.expressions.core.Expr]]]
grains: List[sqlglot.expressions.core.Expr]
references: List[sqlglot.expressions.core.Expr]
physical_schema_override: Optional[str]
physical_properties_: Optional[sqlglot.expressions.query.Tuple]
virtual_properties_: Optional[sqlglot.expressions.query.Tuple]
session_properties_: Optional[sqlglot.expressions.query.Tuple]
allow_partials: bool
signals: List[Tuple[str, Dict[str, sqlglot.expressions.core.Expr]]]
enabled: bool
physical_version: Optional[str]
gateway: Optional[str]
optimize_query: Optional[bool]
ignored_rules_: Optional[Set[str]]
formatting: Optional[bool]
grants_: Optional[sqlglot.expressions.query.Tuple]
grants_target_layer: GrantsTargetLayer
@field_validator('ignored_rules_', mode='before')
def ignored_rules_validator(cls, vs: Any) -> Any:
374    @field_validator("ignored_rules_", mode="before")
375    def ignored_rules_validator(cls, vs: t.Any) -> t.Any:
376        return LinterConfig._validate_rules(vs)
@field_validator('session_properties_', mode='before')
def session_properties_validator(cls, v: Any, info: pydantic_core.core_schema.ValidationInfo) -> Any:
386    @field_validator("session_properties_", mode="before")
387    def session_properties_validator(cls, v: t.Any, info: ValidationInfo) -> t.Any:
388        # use the generic properties validator to parse the session properties
389        parsed_session_properties = parse_properties(type(cls), v, info)
390        if not parsed_session_properties:
391            return parsed_session_properties
392
393        for eq in parsed_session_properties:
394            prop_name = eq.left.name
395
396            if prop_name == "query_label":
397                query_label = eq.right
398                if not isinstance(
399                    query_label, (exp.Array, exp.Tuple, exp.Paren, d.MacroFunc, d.MacroVar)
400                ):
401                    raise ConfigError(
402                        "Invalid value for `session_properties.query_label`. Must be an array or tuple."
403                    )
404
405                label_tuples: t.List[exp.Expr] = (
406                    [query_label.unnest()]
407                    if isinstance(query_label, exp.Paren)
408                    else query_label.expressions
409                )
410
411                for label_tuple in label_tuples:
412                    if not (
413                        isinstance(label_tuple, exp.Tuple)
414                        and len(label_tuple.expressions) == 2
415                        and all(isinstance(label, exp.Literal) for label in label_tuple.expressions)
416                    ):
417                        raise ConfigError(
418                            "Invalid entry in `session_properties.query_label`. Must be tuples of string literals with length 2."
419                        )
420            elif prop_name == "authorization":
421                authorization = eq.right
422                if not (
423                    isinstance(authorization, exp.Literal) and authorization.is_string
424                ) and not isinstance(authorization, (d.MacroFunc, d.MacroVar)):
425                    raise ConfigError(
426                        "Invalid value for `session_properties.authorization`. Must be a string literal."
427                    )
428            elif prop_name == "query_tags":
429                query_tags = eq.right
430                if isinstance(query_tags, (d.MacroFunc, d.MacroVar)):
431                    continue
432
433                if not isinstance(query_tags, (exp.Map, exp.VarMap)):
434                    raise ConfigError(
435                        "Invalid value for `session_properties.query_tags`. Must be a map."
436                    )
437
438                keys = query_tags.args.get("keys")
439                values = query_tags.args.get("values")
440                if not isinstance(keys, exp.Array) or not isinstance(values, exp.Array):
441                    raise ConfigError(
442                        "Invalid value for `session_properties.query_tags`. Must be a map with array "
443                        "keys and array values."
444                    )
445
446                for key, value in zip(keys.expressions, values.expressions):
447                    if not isinstance(key, exp.Literal) or not key.is_string:
448                        raise ConfigError(
449                            "Invalid key in `session_properties.query_tags`. Keys must be string literals."
450                        )
451
452                    if not (
453                        isinstance(value, exp.Null)
454                        or (isinstance(value, exp.Literal) and value.is_string)
455                    ):
456                        raise ConfigError(
457                            "Invalid value in `session_properties.query_tags`. Values must be string "
458                            "literals or NULL."
459                        )
460
461        return parsed_session_properties
time_column: Optional[sqlmesh.core.model.kind.TimeColumn]
531    @property
532    def time_column(self) -> t.Optional[TimeColumn]:
533        """The time column for incremental models."""
534        return getattr(self.kind, "time_column", None)

The time column for incremental models.

unique_key: List[sqlglot.expressions.core.Expr]
536    @property
537    def unique_key(self) -> t.List[exp.Expr]:
538        if isinstance(
539            self.kind, (SCDType2ByTimeKind, SCDType2ByColumnKind, IncrementalByUniqueKeyKind)
540        ):
541            return self.kind.unique_key
542        return []
column_descriptions: Dict[str, str]
544    @property
545    def column_descriptions(self) -> t.Dict[str, str]:
546        """A dictionary of column names to annotation comments."""
547        return self.column_descriptions_ or {}

A dictionary of column names to annotation comments.

lookback: int
549    @property
550    def lookback(self) -> int:
551        """The incremental lookback window."""
552        return getattr(self.kind, "lookback", 0) or 0

The incremental lookback window.

def lookback_start( self, start: Union[datetime.date, datetime.datetime, str, int, float]) -> Union[datetime.date, datetime.datetime, str, int, float]:
554    def lookback_start(self, start: TimeLike) -> TimeLike:
555        if self.lookback == 0:
556            return start
557
558        for _ in range(self.lookback):
559            start = self.interval_unit.cron_prev(start)
560        return start
batch_size: Optional[int]
562    @property
563    def batch_size(self) -> t.Optional[int]:
564        """The maximal number of units in a single task for a backfill."""
565        return getattr(self.kind, "batch_size", None)

The maximal number of units in a single task for a backfill.

batch_concurrency: Optional[int]
567    @property
568    def batch_concurrency(self) -> t.Optional[int]:
569        """The maximal number of batches that can run concurrently for a backfill."""
570        return getattr(self.kind, "batch_concurrency", None)

The maximal number of batches that can run concurrently for a backfill.

physical_properties: Dict[str, sqlglot.expressions.core.Expr]
572    @cached_property
573    def physical_properties(self) -> t.Dict[str, exp.Expr]:
574        """A dictionary of properties that will be applied to the physical layer. It replaces table_properties which is deprecated."""
575        if self.physical_properties_:
576            return {e.this.name: e.expression for e in self.physical_properties_.expressions}
577        return {}

A dictionary of properties that will be applied to the physical layer. It replaces table_properties which is deprecated.

virtual_properties: Dict[str, sqlglot.expressions.core.Expr]
579    @cached_property
580    def virtual_properties(self) -> t.Dict[str, exp.Expr]:
581        """A dictionary of properties that will be applied to the virtual layer."""
582        if self.virtual_properties_:
583            return {e.this.name: e.expression for e in self.virtual_properties_.expressions}
584        return {}

A dictionary of properties that will be applied to the virtual layer.

session_properties: Dict[str, sqlglot.expressions.core.Expr | str | int | float | bool]
586    @property
587    def session_properties(self) -> SessionProperties:
588        """A dictionary of session properties."""
589        if not self.session_properties_:
590            return {}
591
592        return d.interpret_key_value_pairs(self.session_properties_)

A dictionary of session properties.

custom_materialization_properties: Dict[str, sqlglot.expressions.core.Expr | str | int | float | bool]
594    @property
595    def custom_materialization_properties(self) -> CustomMaterializationProperties:
596        if isinstance(self.kind, CustomKind):
597            return self.kind.materialization_properties
598        return {}
grants: Optional[<MagicMock id='130969754108736'>]
600    @cached_property
601    def grants(self) -> t.Optional[GrantsConfig]:
602        """A dictionary of grants mapping permission names to lists of grantees."""
603
604        if self.grants_ is None:
605            return None
606
607        if not self.grants_.expressions:
608            return {}
609
610        grants_dict = {}
611        for eq_expr in self.grants_.expressions:
612            try:
613                permission_name = self._validate_config_expression(eq_expr.left)
614                grantee_list = self._validate_nested_config_values(eq_expr.expression)
615                grants_dict[permission_name] = grantee_list
616            except ConfigError as e:
617                permission_name = (
618                    eq_expr.left.name if hasattr(eq_expr.left, "name") else str(eq_expr.left)
619                )
620                raise ConfigError(f"Invalid grants configuration for '{permission_name}': {e}")
621
622        return grants_dict if grants_dict else None

A dictionary of grants mapping permission names to lists of grantees.

all_references: List[sqlmesh.core.reference.Reference]
624    @property
625    def all_references(self) -> t.List[Reference]:
626        """All references including grains."""
627        return [Reference(model_name=self.name, expression=e, unique=True) for e in self.grains] + [
628            Reference(model_name=self.name, expression=e, unique=True) for e in self.references
629        ]

All references including grains.

on: List[str]
631    @property
632    def on(self) -> t.List[str]:
633        """The grains to be used as join condition in table_diff."""
634
635        on: t.List[str] = []
636        for expr in [ref.expression for ref in self.all_references if ref.unique]:
637            if isinstance(expr, exp.Tuple):
638                on.extend([key.this.sql(dialect=self.dialect) for key in expr.expressions])
639            else:
640                # Handle a single Column or Paren expression
641                on.append(expr.this.sql(dialect=self.dialect))
642
643        return on

The grains to be used as join condition in table_diff.

managed_columns: Dict[str, sqlglot.expressions.datatypes.DataType]
645    @property
646    def managed_columns(self) -> t.Dict[str, exp.DataType]:
647        return getattr(self.kind, "managed_columns", {})
when_matched: Optional[sqlglot.expressions.dml.Whens]
649    @property
650    def when_matched(self) -> t.Optional[exp.Whens]:
651        if isinstance(self.kind, IncrementalByUniqueKeyKind):
652            return self.kind.when_matched
653        return None
merge_filter: Optional[sqlglot.expressions.core.Expr]
655    @property
656    def merge_filter(self) -> t.Optional[exp.Expr]:
657        if isinstance(self.kind, IncrementalByUniqueKeyKind):
658            return self.kind.merge_filter
659        return None
catalog: Optional[str]
661    @property
662    def catalog(self) -> t.Optional[str]:
663        """Returns the catalog of a model."""
664        return self.fully_qualified_table.catalog

Returns the catalog of a model.

fully_qualified_table: sqlglot.expressions.query.Table
666    @cached_property
667    def fully_qualified_table(self) -> exp.Table:
668        return exp.to_table(self.fqn)
fqn: str
670    @cached_property
671    def fqn(self) -> str:
672        return normalize_model_name(
673            self.name, default_catalog=self.default_catalog, dialect=self.dialect
674        )
on_destructive_change: sqlmesh.core.model.kind.OnDestructiveChange
676    @property
677    def on_destructive_change(self) -> OnDestructiveChange:
678        return getattr(self.kind, "on_destructive_change", OnDestructiveChange.ALLOW)
on_additive_change: sqlmesh.core.model.kind.OnAdditiveChange
680    @property
681    def on_additive_change(self) -> OnAdditiveChange:
682        """Return the model's additive change setting if it has one."""
683        return getattr(self.kind, "on_additive_change", OnAdditiveChange.ALLOW)

Return the model's additive change setting if it has one.

ignored_rules: Set[str]
685    @property
686    def ignored_rules(self) -> t.Set[str]:
687        return self.ignored_rules_ or set()
model_config = {'json_encoders': {<class 'sqlglot.expressions.core.Expr'>: <function _expression_encoder>, <class 'sqlglot.expressions.datatypes.DataType'>: <function _expression_encoder>, <class 'sqlglot.expressions.query.Tuple'>: <function _expression_encoder>, typing.Union[sqlglot.expressions.query.Query, sqlmesh.core.dialect.JinjaQuery]: <function _expression_encoder>, typing.Union[sqlglot.expressions.query.Query, sqlmesh.core.dialect.JinjaQuery, sqlmesh.core.dialect.MacroFunc]: <function _expression_encoder>, <class 'datetime.tzinfo'>: <function PydanticModel.<lambda>>}, 'arbitrary_types_allowed': True, 'extra': 'forbid', 'protected_namespaces': ()}

Configuration for the model, should be a dictionary conforming to [ConfigDict][pydantic.config.ConfigDict].

def model_post_init(self: pydantic.main.BaseModel, context: Any, /) -> None:
365def init_private_attributes(self: BaseModel, context: Any, /) -> None:
366    """This function is meant to behave like a BaseModel method to initialize private attributes.
367
368    It takes context as an argument since that's what pydantic-core passes when calling it.
369
370    Args:
371        self: The BaseModel instance.
372        context: The context.
373    """
374    if getattr(self, '__pydantic_private__', None) is None:
375        pydantic_private = {}
376        for name, private_attr in self.__private_attributes__.items():
377            # Avoid needlessly creating a new dict for the validated data:
378            if private_attr.default_factory_takes_validated_data:
379                default = private_attr.get_default(
380                    call_default_factory=True, validated_data={**self.__dict__, **pydantic_private}
381                )
382            else:
383                default = private_attr.get_default(call_default_factory=True)
384            if default is not PydanticUndefined:
385                pydantic_private[name] = default
386        object_setattr(self, '__pydantic_private__', pydantic_private)

This function is meant to behave like a BaseModel method to initialize private attributes.

It takes context as an argument since that's what pydantic-core passes when calling it.

Arguments:
  • self: The BaseModel instance.
  • context: The context.
Inherited Members
pydantic.main.BaseModel
BaseModel
model_fields
model_computed_fields
model_extra
model_fields_set
model_construct
model_copy
model_dump
model_dump_json
model_json_schema
model_parametrized_name
model_rebuild
model_validate
model_validate_json
model_validate_strings
parse_file
from_orm
construct
schema
schema_json
validate
update_forward_refs
sqlmesh.core.node._Node
project
description
owner
start
end
cron
cron_tz
interval_unit_
tags
stamp
dbt_node_info_
copy
interval_unit
depends_on
data_hash
metadata_hash
is_metadata_only_change
is_data_change
croniter
cron_next
cron_prev
cron_floor
text_diff
is_model
is_audit
dbt_node_info
sqlmesh.core.node.DbtInfoMixin
dbt_unique_id
dbt_fqn
sqlmesh.utils.pydantic.PydanticModel
dict
json
fields_set
parse_obj
parse_raw
missing_required_fields
extra_fields
all_fields
all_field_infos
required_fields