Edit on GitHub

sqlmesh.core.model.kind

  1from __future__ import annotations
  2
  3import sys
  4import typing as t
  5from enum import Enum
  6
  7from pydantic import Field
  8from sqlglot import exp
  9from sqlglot.optimizer.normalize_identifiers import normalize_identifiers
 10from sqlglot.optimizer.qualify_columns import quote_identifiers
 11from sqlglot.optimizer.simplify import gen
 12from sqlglot.time import format_time
 13
 14from sqlmesh.core import dialect as d
 15from sqlmesh.core.model.common import parse_properties
 16from sqlmesh.core.model.seed import CsvSettings
 17from sqlmesh.utils.errors import ConfigError
 18from sqlmesh.utils.pydantic import (
 19    PydanticModel,
 20    SQLGlotBool,
 21    SQLGlotColumn,
 22    SQLGlotListOfColumnsOrStar,
 23    SQLGlotListOfFields,
 24    SQLGlotPositiveInt,
 25    SQLGlotString,
 26    column_validator,
 27    field_validator,
 28    field_validator_v1_args,
 29    get_dialect,
 30)
 31
 32if sys.version_info >= (3, 9):
 33    from typing import Annotated, Literal
 34else:
 35    from typing_extensions import Annotated, Literal
 36
 37
 38if t.TYPE_CHECKING:
 39    MODEL_KIND = t.TypeVar("MODEL_KIND", bound="_ModelKind")
 40
 41
 42class ModelKindMixin:
 43    @property
 44    def model_kind_name(self) -> t.Optional[ModelKindName]:
 45        """Returns the model kind name."""
 46        raise NotImplementedError
 47
 48    @property
 49    def is_incremental_by_time_range(self) -> bool:
 50        return self.model_kind_name == ModelKindName.INCREMENTAL_BY_TIME_RANGE
 51
 52    @property
 53    def is_incremental_by_unique_key(self) -> bool:
 54        return self.model_kind_name == ModelKindName.INCREMENTAL_BY_UNIQUE_KEY
 55
 56    @property
 57    def is_incremental_unmanaged(self) -> bool:
 58        return self.model_kind_name == ModelKindName.INCREMENTAL_UNMANAGED
 59
 60    @property
 61    def is_incremental(self) -> bool:
 62        return (
 63            self.is_incremental_by_time_range
 64            or self.is_incremental_by_unique_key
 65            or self.is_incremental_unmanaged
 66            or self.is_scd_type_2
 67        )
 68
 69    @property
 70    def is_full(self) -> bool:
 71        return self.model_kind_name == ModelKindName.FULL
 72
 73    @property
 74    def is_view(self) -> bool:
 75        return self.model_kind_name == ModelKindName.VIEW
 76
 77    @property
 78    def is_embedded(self) -> bool:
 79        return self.model_kind_name == ModelKindName.EMBEDDED
 80
 81    @property
 82    def is_seed(self) -> bool:
 83        return self.model_kind_name == ModelKindName.SEED
 84
 85    @property
 86    def is_external(self) -> bool:
 87        return self.model_kind_name == ModelKindName.EXTERNAL
 88
 89    @property
 90    def is_scd_type_2(self) -> bool:
 91        return self.model_kind_name in {
 92            ModelKindName.SCD_TYPE_2,
 93            ModelKindName.SCD_TYPE_2_BY_TIME,
 94            ModelKindName.SCD_TYPE_2_BY_COLUMN,
 95        }
 96
 97    @property
 98    def is_scd_type_2_by_time(self) -> bool:
 99        return self.model_kind_name in {ModelKindName.SCD_TYPE_2, ModelKindName.SCD_TYPE_2_BY_TIME}
100
101    @property
102    def is_scd_type_2_by_column(self) -> bool:
103        return self.model_kind_name == ModelKindName.SCD_TYPE_2_BY_COLUMN
104
105    @property
106    def is_symbolic(self) -> bool:
107        """A symbolic model is one that doesn't execute at all."""
108        return self.model_kind_name in (ModelKindName.EMBEDDED, ModelKindName.EXTERNAL)
109
110    @property
111    def is_materialized(self) -> bool:
112        return not (self.is_symbolic or self.is_view)
113
114    @property
115    def only_execution_time(self) -> bool:
116        """Whether or not this model only cares about execution time to render."""
117        return self.is_view or self.is_full or self.is_scd_type_2
118
119    @property
120    def full_history_restatement_only(self) -> bool:
121        """Whether or not this model only supports restatement of full history."""
122        return (
123            self.is_incremental_unmanaged or self.is_incremental_by_unique_key or self.is_scd_type_2
124        )
125
126
127class ModelKindName(str, ModelKindMixin, Enum):
128    """The kind of model, determining how this data is computed and stored in the warehouse."""
129
130    INCREMENTAL_BY_TIME_RANGE = "INCREMENTAL_BY_TIME_RANGE"
131    INCREMENTAL_BY_UNIQUE_KEY = "INCREMENTAL_BY_UNIQUE_KEY"
132    INCREMENTAL_UNMANAGED = "INCREMENTAL_UNMANAGED"
133    FULL = "FULL"
134    # Legacy alias to SCD Type 2 By Time
135    # Only used for Parsing and mapping name to SCD Type 2 By Time
136    SCD_TYPE_2 = "SCD_TYPE_2"
137    SCD_TYPE_2_BY_TIME = "SCD_TYPE_2_BY_TIME"
138    SCD_TYPE_2_BY_COLUMN = "SCD_TYPE_2_BY_COLUMN"
139    VIEW = "VIEW"
140    EMBEDDED = "EMBEDDED"
141    SEED = "SEED"
142    EXTERNAL = "EXTERNAL"
143
144    @property
145    def model_kind_name(self) -> t.Optional[ModelKindName]:
146        return self
147
148    def __str__(self) -> str:
149        return self.name
150
151    def __repr__(self) -> str:
152        return str(self)
153
154
155class OnDestructiveChange(str, Enum):
156    """What should happen when a forward-only model change requires a destructive schema change."""
157
158    ERROR = "ERROR"
159    WARN = "WARN"
160    ALLOW = "ALLOW"
161
162    @property
163    def is_error(self) -> bool:
164        return self == OnDestructiveChange.ERROR
165
166    @property
167    def is_warn(self) -> bool:
168        return self == OnDestructiveChange.WARN
169
170    @property
171    def is_allow(self) -> bool:
172        return self == OnDestructiveChange.ALLOW
173
174
175def _on_destructive_change_validator(
176    cls: t.Type, v: t.Union[OnDestructiveChange, str, exp.Identifier]
177) -> t.Any:
178    if v and not isinstance(v, OnDestructiveChange):
179        return OnDestructiveChange(v.this.upper() if isinstance(v, exp.Identifier) else v.upper())
180    return v
181
182
183on_destructive_change_validator = field_validator("on_destructive_change", mode="before")(
184    _on_destructive_change_validator
185)
186
187
188class _ModelKind(PydanticModel, ModelKindMixin):
189    name: ModelKindName
190
191    @property
192    def model_kind_name(self) -> t.Optional[ModelKindName]:
193        return self.name
194
195    def to_expression(self, **kwargs: t.Any) -> d.ModelKind:
196        return d.ModelKind(this=self.name.value.upper(), **kwargs)
197
198    @property
199    def data_hash_values(self) -> t.List[t.Optional[str]]:
200        return [self.name.value]
201
202    @property
203    def metadata_hash_values(self) -> t.List[t.Optional[str]]:
204        return []
205
206
207class TimeColumn(PydanticModel):
208    column: exp.Expression
209    format: t.Optional[str] = None
210
211    @classmethod
212    def validator(cls) -> classmethod:
213        def _time_column_validator(v: t.Any, values: t.Any) -> TimeColumn:
214            dialect = get_dialect(values)
215
216            if isinstance(v, exp.Tuple):
217                column_expr = v.expressions[0]
218                column = (
219                    exp.column(column_expr)
220                    if isinstance(column_expr, exp.Identifier)
221                    else column_expr
222                )
223                format = v.expressions[1].name if len(v.expressions) > 1 else None
224            elif isinstance(v, exp.Expression):
225                column = exp.column(v) if isinstance(v, exp.Identifier) else v
226                format = None
227            elif isinstance(v, str):
228                column = d.parse_one(v, dialect=dialect)
229                column.meta.pop("sql")
230                format = None
231            elif isinstance(v, dict):
232                column_raw = v["column"]
233                column = (
234                    d.parse_one(column_raw, dialect=dialect)
235                    if isinstance(column_raw, str)
236                    else column_raw
237                )
238                format = v.get("format")
239            elif isinstance(v, TimeColumn):
240                column = v.column
241                format = v.format
242            else:
243                raise ConfigError(f"Invalid time_column: '{v}'.")
244
245            column = quote_identifiers(
246                normalize_identifiers(column, dialect=dialect), dialect=dialect
247            )
248            column.meta["dialect"] = dialect
249
250            return TimeColumn(column=column, format=format)
251
252        return field_validator("time_column", mode="before")(_time_column_validator)
253
254    @field_validator("column", mode="before")
255    @classmethod
256    def _column_validator(cls, v: t.Union[str, exp.Expression]) -> exp.Expression:
257        if not v:
258            raise ConfigError("Time Column cannot be empty.")
259        if isinstance(v, str):
260            return exp.to_column(v)
261        return v
262
263    @property
264    def expression(self) -> exp.Expression:
265        """Convert this pydantic model into a time_column SQLGlot expression."""
266        if not self.format:
267            return self.column
268
269        return exp.Tuple(expressions=[self.column, exp.Literal.string(self.format)])
270
271    def to_expression(self, dialect: str) -> exp.Expression:
272        """Convert this pydantic model into a time_column SQLGlot expression."""
273        if not self.format:
274            return self.column
275
276        return exp.Tuple(
277            expressions=[
278                self.column,
279                exp.Literal.string(
280                    format_time(self.format, d.Dialect.get_or_raise(dialect).INVERSE_TIME_MAPPING)
281                ),
282            ]
283        )
284
285    def to_property(self, dialect: str = "") -> exp.Property:
286        return exp.Property(this="time_column", value=self.to_expression(dialect))
287
288
289def _kind_dialect_validator(cls: t.Type, v: t.Optional[str]) -> str:
290    if v is None:
291        return get_dialect({})
292    return v
293
294
295kind_dialect_validator = field_validator("dialect", mode="before", always=True)(
296    _kind_dialect_validator
297)
298
299
300class _Incremental(_ModelKind):
301    on_destructive_change: OnDestructiveChange = OnDestructiveChange.ERROR
302
303    _on_destructive_change_validator = on_destructive_change_validator
304
305    @property
306    def metadata_hash_values(self) -> t.List[t.Optional[str]]:
307        return [
308            *super().metadata_hash_values,
309            str(self.on_destructive_change),
310        ]
311
312
313class _IncrementalBy(_Incremental):
314    dialect: t.Optional[str] = Field(None, validate_default=True)
315    batch_size: t.Optional[SQLGlotPositiveInt] = None
316    batch_concurrency: t.Optional[SQLGlotPositiveInt] = None
317    lookback: t.Optional[SQLGlotPositiveInt] = None
318    forward_only: SQLGlotBool = False
319    disable_restatement: SQLGlotBool = False
320
321    _dialect_validator = kind_dialect_validator
322
323    @property
324    def data_hash_values(self) -> t.List[t.Optional[str]]:
325        return [
326            *super().data_hash_values,
327            self.dialect,
328            str(self.lookback) if self.lookback is not None else None,
329        ]
330
331    @property
332    def metadata_hash_values(self) -> t.List[t.Optional[str]]:
333        return [
334            *super().metadata_hash_values,
335            str(self.batch_size) if self.batch_size is not None else None,
336            str(self.forward_only),
337            str(self.disable_restatement),
338        ]
339
340
341class IncrementalByTimeRangeKind(_IncrementalBy):
342    name: Literal[ModelKindName.INCREMENTAL_BY_TIME_RANGE] = ModelKindName.INCREMENTAL_BY_TIME_RANGE
343    time_column: TimeColumn
344
345    _time_column_validator = TimeColumn.validator()
346
347    def to_expression(self, dialect: str = "", **kwargs: t.Any) -> d.ModelKind:
348        return super().to_expression(expressions=[self.time_column.to_property(dialect)])
349
350    @property
351    def data_hash_values(self) -> t.List[t.Optional[str]]:
352        return [*super().data_hash_values, gen(self.time_column.column), self.time_column.format]
353
354
355class IncrementalByUniqueKeyKind(_IncrementalBy):
356    name: Literal[ModelKindName.INCREMENTAL_BY_UNIQUE_KEY] = ModelKindName.INCREMENTAL_BY_UNIQUE_KEY
357    unique_key: SQLGlotListOfFields
358    when_matched: t.Optional[exp.When] = None
359    batch_concurrency: Literal[1] = 1
360
361    @field_validator("when_matched", mode="before")
362    @field_validator_v1_args
363    def _when_matched_validator(
364        cls, v: t.Optional[t.Union[exp.When, str]], values: t.Dict[str, t.Any]
365    ) -> t.Optional[exp.When]:
366        def replace_table_references(expression: exp.Expression) -> exp.Expression:
367            from sqlmesh.core.engine_adapter.base import (
368                MERGE_SOURCE_ALIAS,
369                MERGE_TARGET_ALIAS,
370            )
371
372            if isinstance(expression, exp.Column):
373                if expression.table.lower() == "target":
374                    expression.set(
375                        "table",
376                        exp.to_identifier(MERGE_TARGET_ALIAS),
377                    )
378                elif expression.table.lower() == "source":
379                    expression.set(
380                        "table",
381                        exp.to_identifier(MERGE_SOURCE_ALIAS),
382                    )
383            return expression
384
385        if isinstance(v, str):
386            return t.cast(exp.When, d.parse_one(v, into=exp.When, dialect=get_dialect(values)))
387
388        if not v:
389            return v
390
391        return t.cast(exp.When, v.transform(replace_table_references))
392
393    @property
394    def data_hash_values(self) -> t.List[t.Optional[str]]:
395        return [
396            *super().data_hash_values,
397            *(gen(k) for k in self.unique_key),
398            gen(self.when_matched) if self.when_matched is not None else None,
399        ]
400
401
402class IncrementalUnmanagedKind(_Incremental):
403    name: Literal[ModelKindName.INCREMENTAL_UNMANAGED] = ModelKindName.INCREMENTAL_UNMANAGED
404    insert_overwrite: SQLGlotBool = False
405    forward_only: SQLGlotBool = True
406    disable_restatement: SQLGlotBool = True
407
408    @property
409    def data_hash_values(self) -> t.List[t.Optional[str]]:
410        return [*super().data_hash_values, str(self.insert_overwrite)]
411
412    @property
413    def metadata_hash_values(self) -> t.List[t.Optional[str]]:
414        return [*super().metadata_hash_values, str(self.forward_only)]
415
416
417class ViewKind(_ModelKind):
418    name: Literal[ModelKindName.VIEW] = ModelKindName.VIEW
419    materialized: SQLGlotBool = False
420
421    @property
422    def data_hash_values(self) -> t.List[t.Optional[str]]:
423        return [*super().data_hash_values, str(self.materialized)]
424
425
426class SeedKind(_ModelKind):
427    name: Literal[ModelKindName.SEED] = ModelKindName.SEED
428    path: SQLGlotString
429    batch_size: SQLGlotPositiveInt = 1000
430    csv_settings: t.Optional[CsvSettings] = None
431
432    @field_validator("csv_settings", mode="before")
433    @classmethod
434    def _parse_csv_settings(cls, v: t.Any) -> t.Optional[CsvSettings]:
435        if v is None or isinstance(v, CsvSettings):
436            return v
437        if isinstance(v, exp.Expression):
438            tuple_exp = parse_properties(cls, v, {})
439            if not tuple_exp:
440                return None
441            return CsvSettings(**{e.left.name: e.right for e in tuple_exp.expressions})
442        if isinstance(v, dict):
443            return CsvSettings(**v)
444        return v
445
446    def to_expression(self, **kwargs: t.Any) -> d.ModelKind:
447        """Convert the seed kind into a SQLGlot expression."""
448        return super().to_expression(
449            expressions=[
450                exp.Property(this=exp.Var(this="path"), value=exp.Literal.string(self.path)),
451                exp.Property(
452                    this=exp.Var(this="batch_size"),
453                    value=exp.Literal.number(self.batch_size),
454                ),
455            ],
456        )
457
458    @property
459    def data_hash_values(self) -> t.List[t.Optional[str]]:
460        return [
461            *super().data_hash_values,
462            *(self.csv_settings or CsvSettings()).dict().values(),
463        ]
464
465    @property
466    def metadata_hash_values(self) -> t.List[t.Optional[str]]:
467        return [*super().metadata_hash_values, str(self.batch_size)]
468
469
470class FullKind(_ModelKind):
471    name: Literal[ModelKindName.FULL] = ModelKindName.FULL
472
473
474class _SCDType2Kind(_Incremental):
475    dialect: t.Optional[str] = Field(None, validate_default=True)
476    unique_key: SQLGlotListOfFields
477    valid_from_name: SQLGlotColumn = Field(exp.column("valid_from"), validate_default=True)
478    valid_to_name: SQLGlotColumn = Field(exp.column("valid_to"), validate_default=True)
479    invalidate_hard_deletes: SQLGlotBool = False
480    time_data_type: exp.DataType = Field(exp.DataType.build("TIMESTAMP"), validate_default=True)
481
482    forward_only: SQLGlotBool = True
483    disable_restatement: SQLGlotBool = True
484
485    _dialect_validator = kind_dialect_validator
486
487    # Remove once Pydantic 1 is deprecated
488    _always_validate_column = field_validator(
489        "valid_from_name", "valid_to_name", mode="before", always=True
490    )(column_validator)
491
492    # always=True can be removed once Pydantic 1 is deprecated
493    @field_validator("time_data_type", mode="before", always=True)
494    @classmethod
495    def _time_data_type_validator(
496        cls, v: t.Union[str, exp.Expression], values: t.Any
497    ) -> exp.Expression:
498        if isinstance(v, exp.Expression) and not isinstance(v, exp.DataType):
499            v = v.name
500        dialect = get_dialect(values)
501        data_type = exp.DataType.build(v, dialect=dialect)
502        data_type.meta["dialect"] = dialect
503        return data_type
504
505    @property
506    def managed_columns(self) -> t.Dict[str, exp.DataType]:
507        return {
508            self.valid_from_name.name: self.time_data_type,
509            self.valid_to_name.name: self.time_data_type,
510        }
511
512    @property
513    def data_hash_values(self) -> t.List[t.Optional[str]]:
514        return [
515            *super().data_hash_values,
516            self.dialect,
517            *(gen(k) for k in self.unique_key),
518            gen(self.valid_from_name),
519            gen(self.valid_to_name),
520            str(self.invalidate_hard_deletes),
521            gen(self.time_data_type),
522        ]
523
524    @property
525    def metadata_hash_values(self) -> t.List[t.Optional[str]]:
526        return [
527            *super().metadata_hash_values,
528            str(self.forward_only),
529            str(self.disable_restatement),
530        ]
531
532
533class SCDType2ByTimeKind(_SCDType2Kind):
534    name: Literal[ModelKindName.SCD_TYPE_2, ModelKindName.SCD_TYPE_2_BY_TIME] = (
535        ModelKindName.SCD_TYPE_2_BY_TIME
536    )
537    updated_at_name: SQLGlotColumn = Field(exp.column("updated_at"), validate_default=True)
538    updated_at_as_valid_from: SQLGlotBool = False
539
540    # Remove once Pydantic 1 is deprecated
541    _always_validate_updated_at = field_validator("updated_at_name", mode="before", always=True)(
542        column_validator
543    )
544
545    @property
546    def data_hash_values(self) -> t.List[t.Optional[str]]:
547        return [
548            *super().data_hash_values,
549            gen(self.updated_at_name),
550            str(self.updated_at_as_valid_from),
551        ]
552
553
554class SCDType2ByColumnKind(_SCDType2Kind):
555    name: Literal[ModelKindName.SCD_TYPE_2_BY_COLUMN] = ModelKindName.SCD_TYPE_2_BY_COLUMN
556    columns: SQLGlotListOfColumnsOrStar
557    execution_time_as_valid_from: SQLGlotBool = False
558
559    @property
560    def data_hash_values(self) -> t.List[t.Optional[str]]:
561        columns_sql = (
562            [gen(c) for c in self.columns]
563            if isinstance(self.columns, list)
564            else [gen(self.columns)]
565        )
566        return [*super().data_hash_values, *columns_sql, str(self.execution_time_as_valid_from)]
567
568
569class EmbeddedKind(_ModelKind):
570    name: Literal[ModelKindName.EMBEDDED] = ModelKindName.EMBEDDED
571
572
573class ExternalKind(_ModelKind):
574    name: Literal[ModelKindName.EXTERNAL] = ModelKindName.EXTERNAL
575
576
577ModelKind = Annotated[
578    t.Union[
579        EmbeddedKind,
580        ExternalKind,
581        FullKind,
582        IncrementalByTimeRangeKind,
583        IncrementalByUniqueKeyKind,
584        IncrementalUnmanagedKind,
585        SeedKind,
586        ViewKind,
587        SCDType2ByTimeKind,
588        SCDType2ByColumnKind,
589    ],
590    Field(discriminator="name"),
591]
592
593MODEL_KIND_NAME_TO_TYPE: t.Dict[str, t.Type[ModelKind]] = {
594    ModelKindName.EMBEDDED: EmbeddedKind,
595    ModelKindName.EXTERNAL: ExternalKind,
596    ModelKindName.FULL: FullKind,
597    ModelKindName.INCREMENTAL_BY_TIME_RANGE: IncrementalByTimeRangeKind,
598    ModelKindName.INCREMENTAL_BY_UNIQUE_KEY: IncrementalByUniqueKeyKind,
599    ModelKindName.INCREMENTAL_UNMANAGED: IncrementalUnmanagedKind,
600    ModelKindName.SEED: SeedKind,
601    ModelKindName.VIEW: ViewKind,
602    ModelKindName.SCD_TYPE_2: SCDType2ByTimeKind,
603    ModelKindName.SCD_TYPE_2_BY_TIME: SCDType2ByTimeKind,
604    ModelKindName.SCD_TYPE_2_BY_COLUMN: SCDType2ByColumnKind,
605}
606
607
608def model_kind_type_from_name(name: t.Optional[str]) -> t.Type[ModelKind]:
609    klass = MODEL_KIND_NAME_TO_TYPE.get(name) if name else None
610    if not klass:
611        raise ConfigError(f"Invalid model kind '{name}'")
612    return t.cast(t.Type[ModelKind], klass)
613
614
615def create_model_kind(v: t.Any, dialect: str, defaults: t.Dict[str, t.Any]) -> ModelKind:
616    if isinstance(v, _ModelKind):
617        return t.cast(ModelKind, v)
618
619    if isinstance(v, (d.ModelKind, dict)):
620        props = (
621            {prop.name: prop.args.get("value") for prop in v.expressions}
622            if isinstance(v, d.ModelKind)
623            else v
624        )
625        name = v.this if isinstance(v, d.ModelKind) else props.get("name")
626
627        # We want to ensure whatever name is provided to construct the class is the same name that will be
628        # found inside the class itself in order to avoid a change during plan/apply for legacy aliases.
629        # Ex: Pass in `SCD_TYPE_2` then we want to ensure we get `SCD_TYPE_2` as the kind name
630        # instead of `SCD_TYPE_2_BY_TIME`.
631        props["name"] = name
632        kind_type = model_kind_type_from_name(name)
633
634        if "dialect" in kind_type.all_fields() and props.get("dialect") is None:
635            props["dialect"] = dialect
636
637        # only pass the on_destructive_change user default to models inheriting from _Incremental
638        # that don't explicitly set it in the model definition
639        if (
640            issubclass(kind_type, _Incremental)
641            and props.get("on_destructive_change") is None
642            and defaults.get("on_destructive_change") is not None
643        ):
644            props["on_destructive_change"] = defaults.get("on_destructive_change")
645
646        return kind_type(**props)
647
648    name = (v.name if isinstance(v, exp.Expression) else str(v)).upper()
649    return model_kind_type_from_name(name)(name=name)  # type: ignore
650
651
652@field_validator_v1_args
653def _model_kind_validator(cls: t.Type, v: t.Any, values: t.Dict[str, t.Any]) -> ModelKind:
654    dialect = get_dialect(values)
655    return create_model_kind(v, dialect, {})
656
657
658model_kind_validator = field_validator("kind", mode="before")(_model_kind_validator)
class ModelKindMixin:
 43class ModelKindMixin:
 44    @property
 45    def model_kind_name(self) -> t.Optional[ModelKindName]:
 46        """Returns the model kind name."""
 47        raise NotImplementedError
 48
 49    @property
 50    def is_incremental_by_time_range(self) -> bool:
 51        return self.model_kind_name == ModelKindName.INCREMENTAL_BY_TIME_RANGE
 52
 53    @property
 54    def is_incremental_by_unique_key(self) -> bool:
 55        return self.model_kind_name == ModelKindName.INCREMENTAL_BY_UNIQUE_KEY
 56
 57    @property
 58    def is_incremental_unmanaged(self) -> bool:
 59        return self.model_kind_name == ModelKindName.INCREMENTAL_UNMANAGED
 60
 61    @property
 62    def is_incremental(self) -> bool:
 63        return (
 64            self.is_incremental_by_time_range
 65            or self.is_incremental_by_unique_key
 66            or self.is_incremental_unmanaged
 67            or self.is_scd_type_2
 68        )
 69
 70    @property
 71    def is_full(self) -> bool:
 72        return self.model_kind_name == ModelKindName.FULL
 73
 74    @property
 75    def is_view(self) -> bool:
 76        return self.model_kind_name == ModelKindName.VIEW
 77
 78    @property
 79    def is_embedded(self) -> bool:
 80        return self.model_kind_name == ModelKindName.EMBEDDED
 81
 82    @property
 83    def is_seed(self) -> bool:
 84        return self.model_kind_name == ModelKindName.SEED
 85
 86    @property
 87    def is_external(self) -> bool:
 88        return self.model_kind_name == ModelKindName.EXTERNAL
 89
 90    @property
 91    def is_scd_type_2(self) -> bool:
 92        return self.model_kind_name in {
 93            ModelKindName.SCD_TYPE_2,
 94            ModelKindName.SCD_TYPE_2_BY_TIME,
 95            ModelKindName.SCD_TYPE_2_BY_COLUMN,
 96        }
 97
 98    @property
 99    def is_scd_type_2_by_time(self) -> bool:
100        return self.model_kind_name in {ModelKindName.SCD_TYPE_2, ModelKindName.SCD_TYPE_2_BY_TIME}
101
102    @property
103    def is_scd_type_2_by_column(self) -> bool:
104        return self.model_kind_name == ModelKindName.SCD_TYPE_2_BY_COLUMN
105
106    @property
107    def is_symbolic(self) -> bool:
108        """A symbolic model is one that doesn't execute at all."""
109        return self.model_kind_name in (ModelKindName.EMBEDDED, ModelKindName.EXTERNAL)
110
111    @property
112    def is_materialized(self) -> bool:
113        return not (self.is_symbolic or self.is_view)
114
115    @property
116    def only_execution_time(self) -> bool:
117        """Whether or not this model only cares about execution time to render."""
118        return self.is_view or self.is_full or self.is_scd_type_2
119
120    @property
121    def full_history_restatement_only(self) -> bool:
122        """Whether or not this model only supports restatement of full history."""
123        return (
124            self.is_incremental_unmanaged or self.is_incremental_by_unique_key or self.is_scd_type_2
125        )
model_kind_name: Union[sqlmesh.core.model.kind.ModelKindName, NoneType]

Returns the model kind name.

is_symbolic: bool

A symbolic model is one that doesn't execute at all.

only_execution_time: bool

Whether or not this model only cares about execution time to render.

full_history_restatement_only: bool

Whether or not this model only supports restatement of full history.

class ModelKindName(builtins.str, ModelKindMixin, enum.Enum):
128class ModelKindName(str, ModelKindMixin, Enum):
129    """The kind of model, determining how this data is computed and stored in the warehouse."""
130
131    INCREMENTAL_BY_TIME_RANGE = "INCREMENTAL_BY_TIME_RANGE"
132    INCREMENTAL_BY_UNIQUE_KEY = "INCREMENTAL_BY_UNIQUE_KEY"
133    INCREMENTAL_UNMANAGED = "INCREMENTAL_UNMANAGED"
134    FULL = "FULL"
135    # Legacy alias to SCD Type 2 By Time
136    # Only used for Parsing and mapping name to SCD Type 2 By Time
137    SCD_TYPE_2 = "SCD_TYPE_2"
138    SCD_TYPE_2_BY_TIME = "SCD_TYPE_2_BY_TIME"
139    SCD_TYPE_2_BY_COLUMN = "SCD_TYPE_2_BY_COLUMN"
140    VIEW = "VIEW"
141    EMBEDDED = "EMBEDDED"
142    SEED = "SEED"
143    EXTERNAL = "EXTERNAL"
144
145    @property
146    def model_kind_name(self) -> t.Optional[ModelKindName]:
147        return self
148
149    def __str__(self) -> str:
150        return self.name
151
152    def __repr__(self) -> str:
153        return str(self)

The kind of model, determining how this data is computed and stored in the warehouse.

INCREMENTAL_BY_TIME_RANGE = INCREMENTAL_BY_TIME_RANGE
INCREMENTAL_BY_UNIQUE_KEY = INCREMENTAL_BY_UNIQUE_KEY
INCREMENTAL_UNMANAGED = INCREMENTAL_UNMANAGED
FULL = FULL
SCD_TYPE_2 = SCD_TYPE_2
SCD_TYPE_2_BY_TIME = SCD_TYPE_2_BY_TIME
SCD_TYPE_2_BY_COLUMN = SCD_TYPE_2_BY_COLUMN
VIEW = VIEW
EMBEDDED = EMBEDDED
SEED = SEED
EXTERNAL = EXTERNAL
model_kind_name: Union[sqlmesh.core.model.kind.ModelKindName, NoneType]

Returns the model kind name.

Inherited Members
ModelKindMixin
is_symbolic
only_execution_time
full_history_restatement_only
enum.Enum
name
value
builtins.str
encode
replace
split
rsplit
join
capitalize
casefold
title
center
count
expandtabs
find
partition
index
ljust
lower
lstrip
rfind
rindex
rjust
rstrip
rpartition
splitlines
strip
swapcase
translate
upper
startswith
endswith
isascii
islower
isupper
istitle
isspace
isdecimal
isdigit
isnumeric
isalpha
isalnum
isidentifier
isprintable
zfill
format
format_map
maketrans
class OnDestructiveChange(builtins.str, enum.Enum):
156class OnDestructiveChange(str, Enum):
157    """What should happen when a forward-only model change requires a destructive schema change."""
158
159    ERROR = "ERROR"
160    WARN = "WARN"
161    ALLOW = "ALLOW"
162
163    @property
164    def is_error(self) -> bool:
165        return self == OnDestructiveChange.ERROR
166
167    @property
168    def is_warn(self) -> bool:
169        return self == OnDestructiveChange.WARN
170
171    @property
172    def is_allow(self) -> bool:
173        return self == OnDestructiveChange.ALLOW

What should happen when a forward-only model change requires a destructive schema change.

ERROR = <OnDestructiveChange.ERROR: 'ERROR'>
WARN = <OnDestructiveChange.WARN: 'WARN'>
ALLOW = <OnDestructiveChange.ALLOW: 'ALLOW'>
Inherited Members
enum.Enum
name
value
builtins.str
encode
replace
split
rsplit
join
capitalize
casefold
title
center
count
expandtabs
find
partition
index
ljust
lower
lstrip
rfind
rindex
rjust
rstrip
rpartition
splitlines
strip
swapcase
translate
upper
startswith
endswith
isascii
islower
isupper
istitle
isspace
isdecimal
isdigit
isnumeric
isalpha
isalnum
isidentifier
isprintable
zfill
format
format_map
maketrans
def on_destructive_change_validator(unknown):

Wrap a classmethod, staticmethod, property or unbound function and act as a descriptor that allows us to detect decorated items from the class' attributes.

This class' __get__ returns the wrapped item's __get__ result, which makes it transparent for classmethods and staticmethods.

Attributes:
  • wrapped: The decorator that has to be wrapped.
  • decorator_info: The decorator info.
  • shim: A wrapper function to wrap V1 style function.
class TimeColumn(sqlmesh.utils.pydantic.PydanticModel):
208class TimeColumn(PydanticModel):
209    column: exp.Expression
210    format: t.Optional[str] = None
211
212    @classmethod
213    def validator(cls) -> classmethod:
214        def _time_column_validator(v: t.Any, values: t.Any) -> TimeColumn:
215            dialect = get_dialect(values)
216
217            if isinstance(v, exp.Tuple):
218                column_expr = v.expressions[0]
219                column = (
220                    exp.column(column_expr)
221                    if isinstance(column_expr, exp.Identifier)
222                    else column_expr
223                )
224                format = v.expressions[1].name if len(v.expressions) > 1 else None
225            elif isinstance(v, exp.Expression):
226                column = exp.column(v) if isinstance(v, exp.Identifier) else v
227                format = None
228            elif isinstance(v, str):
229                column = d.parse_one(v, dialect=dialect)
230                column.meta.pop("sql")
231                format = None
232            elif isinstance(v, dict):
233                column_raw = v["column"]
234                column = (
235                    d.parse_one(column_raw, dialect=dialect)
236                    if isinstance(column_raw, str)
237                    else column_raw
238                )
239                format = v.get("format")
240            elif isinstance(v, TimeColumn):
241                column = v.column
242                format = v.format
243            else:
244                raise ConfigError(f"Invalid time_column: '{v}'.")
245
246            column = quote_identifiers(
247                normalize_identifiers(column, dialect=dialect), dialect=dialect
248            )
249            column.meta["dialect"] = dialect
250
251            return TimeColumn(column=column, format=format)
252
253        return field_validator("time_column", mode="before")(_time_column_validator)
254
255    @field_validator("column", mode="before")
256    @classmethod
257    def _column_validator(cls, v: t.Union[str, exp.Expression]) -> exp.Expression:
258        if not v:
259            raise ConfigError("Time Column cannot be empty.")
260        if isinstance(v, str):
261            return exp.to_column(v)
262        return v
263
264    @property
265    def expression(self) -> exp.Expression:
266        """Convert this pydantic model into a time_column SQLGlot expression."""
267        if not self.format:
268            return self.column
269
270        return exp.Tuple(expressions=[self.column, exp.Literal.string(self.format)])
271
272    def to_expression(self, dialect: str) -> exp.Expression:
273        """Convert this pydantic model into a time_column SQLGlot expression."""
274        if not self.format:
275            return self.column
276
277        return exp.Tuple(
278            expressions=[
279                self.column,
280                exp.Literal.string(
281                    format_time(self.format, d.Dialect.get_or_raise(dialect).INVERSE_TIME_MAPPING)
282                ),
283            ]
284        )
285
286    def to_property(self, dialect: str = "") -> exp.Property:
287        return exp.Property(this="time_column", value=self.to_expression(dialect))

Usage docs: https://docs.pydantic.dev/2.7/concepts/models/

A base class for creating Pydantic models.

Attributes:
  • __class_vars__: The names of classvars defined on the model.
  • __private_attributes__: Metadata about the private attributes of the model.
  • __signature__: The signature for instantiating the model.
  • __pydantic_complete__: Whether model building is completed, or if there are still undefined fields.
  • __pydantic_core_schema__: The pydantic-core schema used to build the SchemaValidator and SchemaSerializer.
  • __pydantic_custom_init__: Whether the model has a custom __init__ function.
  • __pydantic_decorators__: Metadata containing the decorators defined on the model. This replaces Model.__validators__ and Model.__root_validators__ from Pydantic V1.
  • __pydantic_generic_metadata__: Metadata for generic models; contains data used for a similar purpose to __args__, __origin__, __parameters__ in typing-module generics. May eventually be replaced by these.
  • __pydantic_parent_namespace__: Parent namespace of the model, used for automatic rebuilding of models.
  • __pydantic_post_init__: The name of the post-init method for the model, if defined.
  • __pydantic_root_model__: Whether the model is a RootModel.
  • __pydantic_serializer__: The pydantic-core SchemaSerializer used to dump instances of the model.
  • __pydantic_validator__: The pydantic-core SchemaValidator used to validate instances of the model.
  • __pydantic_extra__: An instance attribute with the values of extra fields from validation when model_config['extra'] == 'allow'.
  • __pydantic_fields_set__: An instance attribute with the names of fields explicitly set.
  • __pydantic_private__: Instance attribute with the values of private attributes set on the model instance.
@classmethod
def validator(cls) -> classmethod:
212    @classmethod
213    def validator(cls) -> classmethod:
214        def _time_column_validator(v: t.Any, values: t.Any) -> TimeColumn:
215            dialect = get_dialect(values)
216
217            if isinstance(v, exp.Tuple):
218                column_expr = v.expressions[0]
219                column = (
220                    exp.column(column_expr)
221                    if isinstance(column_expr, exp.Identifier)
222                    else column_expr
223                )
224                format = v.expressions[1].name if len(v.expressions) > 1 else None
225            elif isinstance(v, exp.Expression):
226                column = exp.column(v) if isinstance(v, exp.Identifier) else v
227                format = None
228            elif isinstance(v, str):
229                column = d.parse_one(v, dialect=dialect)
230                column.meta.pop("sql")
231                format = None
232            elif isinstance(v, dict):
233                column_raw = v["column"]
234                column = (
235                    d.parse_one(column_raw, dialect=dialect)
236                    if isinstance(column_raw, str)
237                    else column_raw
238                )
239                format = v.get("format")
240            elif isinstance(v, TimeColumn):
241                column = v.column
242                format = v.format
243            else:
244                raise ConfigError(f"Invalid time_column: '{v}'.")
245
246            column = quote_identifiers(
247                normalize_identifiers(column, dialect=dialect), dialect=dialect
248            )
249            column.meta["dialect"] = dialect
250
251            return TimeColumn(column=column, format=format)
252
253        return field_validator("time_column", mode="before")(_time_column_validator)
expression: sqlglot.expressions.Expression

Convert this pydantic model into a time_column SQLGlot expression.

def to_expression(self, dialect: str) -> sqlglot.expressions.Expression:
272    def to_expression(self, dialect: str) -> exp.Expression:
273        """Convert this pydantic model into a time_column SQLGlot expression."""
274        if not self.format:
275            return self.column
276
277        return exp.Tuple(
278            expressions=[
279                self.column,
280                exp.Literal.string(
281                    format_time(self.format, d.Dialect.get_or_raise(dialect).INVERSE_TIME_MAPPING)
282                ),
283            ]
284        )

Convert this pydantic model into a time_column SQLGlot expression.

def to_property(self, dialect: str = '') -> sqlglot.expressions.Property:
286    def to_property(self, dialect: str = "") -> exp.Property:
287        return exp.Property(this="time_column", value=self.to_expression(dialect))
Inherited Members
pydantic.main.BaseModel
BaseModel
model_extra
model_fields_set
model_construct
model_copy
model_dump
model_dump_json
model_json_schema
model_parametrized_name
model_rebuild
model_validate
model_validate_json
model_validate_strings
parse_file
from_orm
construct
schema
schema_json
validate
update_forward_refs
sqlmesh.utils.pydantic.PydanticModel
dict
json
copy
parse_obj
parse_raw
missing_required_fields
extra_fields
all_fields
all_field_infos
required_fields
model_post_init
def kind_dialect_validator(unknown):

Wrap a classmethod, staticmethod, property or unbound function and act as a descriptor that allows us to detect decorated items from the class' attributes.

This class' __get__ returns the wrapped item's __get__ result, which makes it transparent for classmethods and staticmethods.

Attributes:
  • wrapped: The decorator that has to be wrapped.
  • decorator_info: The decorator info.
  • shim: A wrapper function to wrap V1 style function.
class IncrementalByTimeRangeKind(_IncrementalBy):
342class IncrementalByTimeRangeKind(_IncrementalBy):
343    name: Literal[ModelKindName.INCREMENTAL_BY_TIME_RANGE] = ModelKindName.INCREMENTAL_BY_TIME_RANGE
344    time_column: TimeColumn
345
346    _time_column_validator = TimeColumn.validator()
347
348    def to_expression(self, dialect: str = "", **kwargs: t.Any) -> d.ModelKind:
349        return super().to_expression(expressions=[self.time_column.to_property(dialect)])
350
351    @property
352    def data_hash_values(self) -> t.List[t.Optional[str]]:
353        return [*super().data_hash_values, gen(self.time_column.column), self.time_column.format]

Usage docs: https://docs.pydantic.dev/2.7/concepts/models/

A base class for creating Pydantic models.

Attributes:
  • __class_vars__: The names of classvars defined on the model.
  • __private_attributes__: Metadata about the private attributes of the model.
  • __signature__: The signature for instantiating the model.
  • __pydantic_complete__: Whether model building is completed, or if there are still undefined fields.
  • __pydantic_core_schema__: The pydantic-core schema used to build the SchemaValidator and SchemaSerializer.
  • __pydantic_custom_init__: Whether the model has a custom __init__ function.
  • __pydantic_decorators__: Metadata containing the decorators defined on the model. This replaces Model.__validators__ and Model.__root_validators__ from Pydantic V1.
  • __pydantic_generic_metadata__: Metadata for generic models; contains data used for a similar purpose to __args__, __origin__, __parameters__ in typing-module generics. May eventually be replaced by these.
  • __pydantic_parent_namespace__: Parent namespace of the model, used for automatic rebuilding of models.
  • __pydantic_post_init__: The name of the post-init method for the model, if defined.
  • __pydantic_root_model__: Whether the model is a RootModel.
  • __pydantic_serializer__: The pydantic-core SchemaSerializer used to dump instances of the model.
  • __pydantic_validator__: The pydantic-core SchemaValidator used to validate instances of the model.
  • __pydantic_extra__: An instance attribute with the values of extra fields from validation when model_config['extra'] == 'allow'.
  • __pydantic_fields_set__: An instance attribute with the names of fields explicitly set.
  • __pydantic_private__: Instance attribute with the values of private attributes set on the model instance.
def to_expression(self, dialect: str = '', **kwargs: Any) -> sqlmesh.core.dialect.ModelKind:
348    def to_expression(self, dialect: str = "", **kwargs: t.Any) -> d.ModelKind:
349        return super().to_expression(expressions=[self.time_column.to_property(dialect)])
Inherited Members
pydantic.main.BaseModel
BaseModel
model_extra
model_fields_set
model_construct
model_copy
model_dump
model_dump_json
model_json_schema
model_parametrized_name
model_rebuild
model_validate
model_validate_json
model_validate_strings
parse_file
from_orm
construct
schema
schema_json
validate
update_forward_refs
_ModelKind
model_kind_name
sqlmesh.utils.pydantic.PydanticModel
dict
json
copy
parse_obj
parse_raw
missing_required_fields
extra_fields
all_fields
all_field_infos
required_fields
model_post_init
ModelKindMixin
is_symbolic
only_execution_time
full_history_restatement_only
class IncrementalByUniqueKeyKind(_IncrementalBy):
356class IncrementalByUniqueKeyKind(_IncrementalBy):
357    name: Literal[ModelKindName.INCREMENTAL_BY_UNIQUE_KEY] = ModelKindName.INCREMENTAL_BY_UNIQUE_KEY
358    unique_key: SQLGlotListOfFields
359    when_matched: t.Optional[exp.When] = None
360    batch_concurrency: Literal[1] = 1
361
362    @field_validator("when_matched", mode="before")
363    @field_validator_v1_args
364    def _when_matched_validator(
365        cls, v: t.Optional[t.Union[exp.When, str]], values: t.Dict[str, t.Any]
366    ) -> t.Optional[exp.When]:
367        def replace_table_references(expression: exp.Expression) -> exp.Expression:
368            from sqlmesh.core.engine_adapter.base import (
369                MERGE_SOURCE_ALIAS,
370                MERGE_TARGET_ALIAS,
371            )
372
373            if isinstance(expression, exp.Column):
374                if expression.table.lower() == "target":
375                    expression.set(
376                        "table",
377                        exp.to_identifier(MERGE_TARGET_ALIAS),
378                    )
379                elif expression.table.lower() == "source":
380                    expression.set(
381                        "table",
382                        exp.to_identifier(MERGE_SOURCE_ALIAS),
383                    )
384            return expression
385
386        if isinstance(v, str):
387            return t.cast(exp.When, d.parse_one(v, into=exp.When, dialect=get_dialect(values)))
388
389        if not v:
390            return v
391
392        return t.cast(exp.When, v.transform(replace_table_references))
393
394    @property
395    def data_hash_values(self) -> t.List[t.Optional[str]]:
396        return [
397            *super().data_hash_values,
398            *(gen(k) for k in self.unique_key),
399            gen(self.when_matched) if self.when_matched is not None else None,
400        ]

Usage docs: https://docs.pydantic.dev/2.7/concepts/models/

A base class for creating Pydantic models.

Attributes:
  • __class_vars__: The names of classvars defined on the model.
  • __private_attributes__: Metadata about the private attributes of the model.
  • __signature__: The signature for instantiating the model.
  • __pydantic_complete__: Whether model building is completed, or if there are still undefined fields.
  • __pydantic_core_schema__: The pydantic-core schema used to build the SchemaValidator and SchemaSerializer.
  • __pydantic_custom_init__: Whether the model has a custom __init__ function.
  • __pydantic_decorators__: Metadata containing the decorators defined on the model. This replaces Model.__validators__ and Model.__root_validators__ from Pydantic V1.
  • __pydantic_generic_metadata__: Metadata for generic models; contains data used for a similar purpose to __args__, __origin__, __parameters__ in typing-module generics. May eventually be replaced by these.
  • __pydantic_parent_namespace__: Parent namespace of the model, used for automatic rebuilding of models.
  • __pydantic_post_init__: The name of the post-init method for the model, if defined.
  • __pydantic_root_model__: Whether the model is a RootModel.
  • __pydantic_serializer__: The pydantic-core SchemaSerializer used to dump instances of the model.
  • __pydantic_validator__: The pydantic-core SchemaValidator used to validate instances of the model.
  • __pydantic_extra__: An instance attribute with the values of extra fields from validation when model_config['extra'] == 'allow'.
  • __pydantic_fields_set__: An instance attribute with the names of fields explicitly set.
  • __pydantic_private__: Instance attribute with the values of private attributes set on the model instance.
Inherited Members
pydantic.main.BaseModel
BaseModel
model_extra
model_fields_set
model_construct
model_copy
model_dump
model_dump_json
model_json_schema
model_parametrized_name
model_rebuild
model_validate
model_validate_json
model_validate_strings
parse_file
from_orm
construct
schema
schema_json
validate
update_forward_refs
_ModelKind
model_kind_name
to_expression
sqlmesh.utils.pydantic.PydanticModel
dict
json
copy
parse_obj
parse_raw
missing_required_fields
extra_fields
all_fields
all_field_infos
required_fields
model_post_init
ModelKindMixin
is_symbolic
only_execution_time
full_history_restatement_only
class IncrementalUnmanagedKind(_Incremental):
403class IncrementalUnmanagedKind(_Incremental):
404    name: Literal[ModelKindName.INCREMENTAL_UNMANAGED] = ModelKindName.INCREMENTAL_UNMANAGED
405    insert_overwrite: SQLGlotBool = False
406    forward_only: SQLGlotBool = True
407    disable_restatement: SQLGlotBool = True
408
409    @property
410    def data_hash_values(self) -> t.List[t.Optional[str]]:
411        return [*super().data_hash_values, str(self.insert_overwrite)]
412
413    @property
414    def metadata_hash_values(self) -> t.List[t.Optional[str]]:
415        return [*super().metadata_hash_values, str(self.forward_only)]

Usage docs: https://docs.pydantic.dev/2.7/concepts/models/

A base class for creating Pydantic models.

Attributes:
  • __class_vars__: The names of classvars defined on the model.
  • __private_attributes__: Metadata about the private attributes of the model.
  • __signature__: The signature for instantiating the model.
  • __pydantic_complete__: Whether model building is completed, or if there are still undefined fields.
  • __pydantic_core_schema__: The pydantic-core schema used to build the SchemaValidator and SchemaSerializer.
  • __pydantic_custom_init__: Whether the model has a custom __init__ function.
  • __pydantic_decorators__: Metadata containing the decorators defined on the model. This replaces Model.__validators__ and Model.__root_validators__ from Pydantic V1.
  • __pydantic_generic_metadata__: Metadata for generic models; contains data used for a similar purpose to __args__, __origin__, __parameters__ in typing-module generics. May eventually be replaced by these.
  • __pydantic_parent_namespace__: Parent namespace of the model, used for automatic rebuilding of models.
  • __pydantic_post_init__: The name of the post-init method for the model, if defined.
  • __pydantic_root_model__: Whether the model is a RootModel.
  • __pydantic_serializer__: The pydantic-core SchemaSerializer used to dump instances of the model.
  • __pydantic_validator__: The pydantic-core SchemaValidator used to validate instances of the model.
  • __pydantic_extra__: An instance attribute with the values of extra fields from validation when model_config['extra'] == 'allow'.
  • __pydantic_fields_set__: An instance attribute with the names of fields explicitly set.
  • __pydantic_private__: Instance attribute with the values of private attributes set on the model instance.
Inherited Members
pydantic.main.BaseModel
BaseModel
model_extra
model_fields_set
model_construct
model_copy
model_dump
model_dump_json
model_json_schema
model_parametrized_name
model_rebuild
model_validate
model_validate_json
model_validate_strings
parse_file
from_orm
construct
schema
schema_json
validate
update_forward_refs
_ModelKind
model_kind_name
to_expression
sqlmesh.utils.pydantic.PydanticModel
dict
json
copy
parse_obj
parse_raw
missing_required_fields
extra_fields
all_fields
all_field_infos
required_fields
model_post_init
ModelKindMixin
is_symbolic
only_execution_time
full_history_restatement_only
class ViewKind(_ModelKind):
418class ViewKind(_ModelKind):
419    name: Literal[ModelKindName.VIEW] = ModelKindName.VIEW
420    materialized: SQLGlotBool = False
421
422    @property
423    def data_hash_values(self) -> t.List[t.Optional[str]]:
424        return [*super().data_hash_values, str(self.materialized)]

Usage docs: https://docs.pydantic.dev/2.7/concepts/models/

A base class for creating Pydantic models.

Attributes:
  • __class_vars__: The names of classvars defined on the model.
  • __private_attributes__: Metadata about the private attributes of the model.
  • __signature__: The signature for instantiating the model.
  • __pydantic_complete__: Whether model building is completed, or if there are still undefined fields.
  • __pydantic_core_schema__: The pydantic-core schema used to build the SchemaValidator and SchemaSerializer.
  • __pydantic_custom_init__: Whether the model has a custom __init__ function.
  • __pydantic_decorators__: Metadata containing the decorators defined on the model. This replaces Model.__validators__ and Model.__root_validators__ from Pydantic V1.
  • __pydantic_generic_metadata__: Metadata for generic models; contains data used for a similar purpose to __args__, __origin__, __parameters__ in typing-module generics. May eventually be replaced by these.
  • __pydantic_parent_namespace__: Parent namespace of the model, used for automatic rebuilding of models.
  • __pydantic_post_init__: The name of the post-init method for the model, if defined.
  • __pydantic_root_model__: Whether the model is a RootModel.
  • __pydantic_serializer__: The pydantic-core SchemaSerializer used to dump instances of the model.
  • __pydantic_validator__: The pydantic-core SchemaValidator used to validate instances of the model.
  • __pydantic_extra__: An instance attribute with the values of extra fields from validation when model_config['extra'] == 'allow'.
  • __pydantic_fields_set__: An instance attribute with the names of fields explicitly set.
  • __pydantic_private__: Instance attribute with the values of private attributes set on the model instance.
Inherited Members
pydantic.main.BaseModel
BaseModel
model_extra
model_fields_set
model_construct
model_copy
model_dump
model_dump_json
model_json_schema
model_parametrized_name
model_rebuild
model_validate
model_validate_json
model_validate_strings
parse_file
from_orm
construct
schema
schema_json
validate
update_forward_refs
_ModelKind
model_kind_name
to_expression
sqlmesh.utils.pydantic.PydanticModel
dict
json
copy
parse_obj
parse_raw
missing_required_fields
extra_fields
all_fields
all_field_infos
required_fields
model_post_init
ModelKindMixin
is_symbolic
only_execution_time
full_history_restatement_only
class SeedKind(_ModelKind):
427class SeedKind(_ModelKind):
428    name: Literal[ModelKindName.SEED] = ModelKindName.SEED
429    path: SQLGlotString
430    batch_size: SQLGlotPositiveInt = 1000
431    csv_settings: t.Optional[CsvSettings] = None
432
433    @field_validator("csv_settings", mode="before")
434    @classmethod
435    def _parse_csv_settings(cls, v: t.Any) -> t.Optional[CsvSettings]:
436        if v is None or isinstance(v, CsvSettings):
437            return v
438        if isinstance(v, exp.Expression):
439            tuple_exp = parse_properties(cls, v, {})
440            if not tuple_exp:
441                return None
442            return CsvSettings(**{e.left.name: e.right for e in tuple_exp.expressions})
443        if isinstance(v, dict):
444            return CsvSettings(**v)
445        return v
446
447    def to_expression(self, **kwargs: t.Any) -> d.ModelKind:
448        """Convert the seed kind into a SQLGlot expression."""
449        return super().to_expression(
450            expressions=[
451                exp.Property(this=exp.Var(this="path"), value=exp.Literal.string(self.path)),
452                exp.Property(
453                    this=exp.Var(this="batch_size"),
454                    value=exp.Literal.number(self.batch_size),
455                ),
456            ],
457        )
458
459    @property
460    def data_hash_values(self) -> t.List[t.Optional[str]]:
461        return [
462            *super().data_hash_values,
463            *(self.csv_settings or CsvSettings()).dict().values(),
464        ]
465
466    @property
467    def metadata_hash_values(self) -> t.List[t.Optional[str]]:
468        return [*super().metadata_hash_values, str(self.batch_size)]

Usage docs: https://docs.pydantic.dev/2.7/concepts/models/

A base class for creating Pydantic models.

Attributes:
  • __class_vars__: The names of classvars defined on the model.
  • __private_attributes__: Metadata about the private attributes of the model.
  • __signature__: The signature for instantiating the model.
  • __pydantic_complete__: Whether model building is completed, or if there are still undefined fields.
  • __pydantic_core_schema__: The pydantic-core schema used to build the SchemaValidator and SchemaSerializer.
  • __pydantic_custom_init__: Whether the model has a custom __init__ function.
  • __pydantic_decorators__: Metadata containing the decorators defined on the model. This replaces Model.__validators__ and Model.__root_validators__ from Pydantic V1.
  • __pydantic_generic_metadata__: Metadata for generic models; contains data used for a similar purpose to __args__, __origin__, __parameters__ in typing-module generics. May eventually be replaced by these.
  • __pydantic_parent_namespace__: Parent namespace of the model, used for automatic rebuilding of models.
  • __pydantic_post_init__: The name of the post-init method for the model, if defined.
  • __pydantic_root_model__: Whether the model is a RootModel.
  • __pydantic_serializer__: The pydantic-core SchemaSerializer used to dump instances of the model.
  • __pydantic_validator__: The pydantic-core SchemaValidator used to validate instances of the model.
  • __pydantic_extra__: An instance attribute with the values of extra fields from validation when model_config['extra'] == 'allow'.
  • __pydantic_fields_set__: An instance attribute with the names of fields explicitly set.
  • __pydantic_private__: Instance attribute with the values of private attributes set on the model instance.
def to_expression(self, **kwargs: Any) -> sqlmesh.core.dialect.ModelKind:
447    def to_expression(self, **kwargs: t.Any) -> d.ModelKind:
448        """Convert the seed kind into a SQLGlot expression."""
449        return super().to_expression(
450            expressions=[
451                exp.Property(this=exp.Var(this="path"), value=exp.Literal.string(self.path)),
452                exp.Property(
453                    this=exp.Var(this="batch_size"),
454                    value=exp.Literal.number(self.batch_size),
455                ),
456            ],
457        )

Convert the seed kind into a SQLGlot expression.

Inherited Members
pydantic.main.BaseModel
BaseModel
model_extra
model_fields_set
model_construct
model_copy
model_dump
model_dump_json
model_json_schema
model_parametrized_name
model_rebuild
model_validate
model_validate_json
model_validate_strings
parse_file
from_orm
construct
schema
schema_json
validate
update_forward_refs
_ModelKind
model_kind_name
sqlmesh.utils.pydantic.PydanticModel
dict
json
copy
parse_obj
parse_raw
missing_required_fields
extra_fields
all_fields
all_field_infos
required_fields
model_post_init
ModelKindMixin
is_symbolic
only_execution_time
full_history_restatement_only
class FullKind(_ModelKind):
471class FullKind(_ModelKind):
472    name: Literal[ModelKindName.FULL] = ModelKindName.FULL

Usage docs: https://docs.pydantic.dev/2.7/concepts/models/

A base class for creating Pydantic models.

Attributes:
  • __class_vars__: The names of classvars defined on the model.
  • __private_attributes__: Metadata about the private attributes of the model.
  • __signature__: The signature for instantiating the model.
  • __pydantic_complete__: Whether model building is completed, or if there are still undefined fields.
  • __pydantic_core_schema__: The pydantic-core schema used to build the SchemaValidator and SchemaSerializer.
  • __pydantic_custom_init__: Whether the model has a custom __init__ function.
  • __pydantic_decorators__: Metadata containing the decorators defined on the model. This replaces Model.__validators__ and Model.__root_validators__ from Pydantic V1.
  • __pydantic_generic_metadata__: Metadata for generic models; contains data used for a similar purpose to __args__, __origin__, __parameters__ in typing-module generics. May eventually be replaced by these.
  • __pydantic_parent_namespace__: Parent namespace of the model, used for automatic rebuilding of models.
  • __pydantic_post_init__: The name of the post-init method for the model, if defined.
  • __pydantic_root_model__: Whether the model is a RootModel.
  • __pydantic_serializer__: The pydantic-core SchemaSerializer used to dump instances of the model.
  • __pydantic_validator__: The pydantic-core SchemaValidator used to validate instances of the model.
  • __pydantic_extra__: An instance attribute with the values of extra fields from validation when model_config['extra'] == 'allow'.
  • __pydantic_fields_set__: An instance attribute with the names of fields explicitly set.
  • __pydantic_private__: Instance attribute with the values of private attributes set on the model instance.
Inherited Members
pydantic.main.BaseModel
BaseModel
model_extra
model_fields_set
model_construct
model_copy
model_dump
model_dump_json
model_json_schema
model_parametrized_name
model_rebuild
model_validate
model_validate_json
model_validate_strings
parse_file
from_orm
construct
schema
schema_json
validate
update_forward_refs
_ModelKind
model_kind_name
to_expression
sqlmesh.utils.pydantic.PydanticModel
dict
json
copy
parse_obj
parse_raw
missing_required_fields
extra_fields
all_fields
all_field_infos
required_fields
model_post_init
ModelKindMixin
is_symbolic
only_execution_time
full_history_restatement_only
class SCDType2ByTimeKind(_SCDType2Kind):
534class SCDType2ByTimeKind(_SCDType2Kind):
535    name: Literal[ModelKindName.SCD_TYPE_2, ModelKindName.SCD_TYPE_2_BY_TIME] = (
536        ModelKindName.SCD_TYPE_2_BY_TIME
537    )
538    updated_at_name: SQLGlotColumn = Field(exp.column("updated_at"), validate_default=True)
539    updated_at_as_valid_from: SQLGlotBool = False
540
541    # Remove once Pydantic 1 is deprecated
542    _always_validate_updated_at = field_validator("updated_at_name", mode="before", always=True)(
543        column_validator
544    )
545
546    @property
547    def data_hash_values(self) -> t.List[t.Optional[str]]:
548        return [
549            *super().data_hash_values,
550            gen(self.updated_at_name),
551            str(self.updated_at_as_valid_from),
552        ]

Usage docs: https://docs.pydantic.dev/2.7/concepts/models/

A base class for creating Pydantic models.

Attributes:
  • __class_vars__: The names of classvars defined on the model.
  • __private_attributes__: Metadata about the private attributes of the model.
  • __signature__: The signature for instantiating the model.
  • __pydantic_complete__: Whether model building is completed, or if there are still undefined fields.
  • __pydantic_core_schema__: The pydantic-core schema used to build the SchemaValidator and SchemaSerializer.
  • __pydantic_custom_init__: Whether the model has a custom __init__ function.
  • __pydantic_decorators__: Metadata containing the decorators defined on the model. This replaces Model.__validators__ and Model.__root_validators__ from Pydantic V1.
  • __pydantic_generic_metadata__: Metadata for generic models; contains data used for a similar purpose to __args__, __origin__, __parameters__ in typing-module generics. May eventually be replaced by these.
  • __pydantic_parent_namespace__: Parent namespace of the model, used for automatic rebuilding of models.
  • __pydantic_post_init__: The name of the post-init method for the model, if defined.
  • __pydantic_root_model__: Whether the model is a RootModel.
  • __pydantic_serializer__: The pydantic-core SchemaSerializer used to dump instances of the model.
  • __pydantic_validator__: The pydantic-core SchemaValidator used to validate instances of the model.
  • __pydantic_extra__: An instance attribute with the values of extra fields from validation when model_config['extra'] == 'allow'.
  • __pydantic_fields_set__: An instance attribute with the names of fields explicitly set.
  • __pydantic_private__: Instance attribute with the values of private attributes set on the model instance.
Inherited Members
pydantic.main.BaseModel
BaseModel
model_extra
model_fields_set
model_construct
model_copy
model_dump
model_dump_json
model_json_schema
model_parametrized_name
model_rebuild
model_validate
model_validate_json
model_validate_strings
parse_file
from_orm
construct
schema
schema_json
validate
update_forward_refs
_ModelKind
model_kind_name
to_expression
sqlmesh.utils.pydantic.PydanticModel
dict
json
copy
parse_obj
parse_raw
missing_required_fields
extra_fields
all_fields
all_field_infos
required_fields
model_post_init
ModelKindMixin
is_symbolic
only_execution_time
full_history_restatement_only
class SCDType2ByColumnKind(_SCDType2Kind):
555class SCDType2ByColumnKind(_SCDType2Kind):
556    name: Literal[ModelKindName.SCD_TYPE_2_BY_COLUMN] = ModelKindName.SCD_TYPE_2_BY_COLUMN
557    columns: SQLGlotListOfColumnsOrStar
558    execution_time_as_valid_from: SQLGlotBool = False
559
560    @property
561    def data_hash_values(self) -> t.List[t.Optional[str]]:
562        columns_sql = (
563            [gen(c) for c in self.columns]
564            if isinstance(self.columns, list)
565            else [gen(self.columns)]
566        )
567        return [*super().data_hash_values, *columns_sql, str(self.execution_time_as_valid_from)]

Usage docs: https://docs.pydantic.dev/2.7/concepts/models/

A base class for creating Pydantic models.

Attributes:
  • __class_vars__: The names of classvars defined on the model.
  • __private_attributes__: Metadata about the private attributes of the model.
  • __signature__: The signature for instantiating the model.
  • __pydantic_complete__: Whether model building is completed, or if there are still undefined fields.
  • __pydantic_core_schema__: The pydantic-core schema used to build the SchemaValidator and SchemaSerializer.
  • __pydantic_custom_init__: Whether the model has a custom __init__ function.
  • __pydantic_decorators__: Metadata containing the decorators defined on the model. This replaces Model.__validators__ and Model.__root_validators__ from Pydantic V1.
  • __pydantic_generic_metadata__: Metadata for generic models; contains data used for a similar purpose to __args__, __origin__, __parameters__ in typing-module generics. May eventually be replaced by these.
  • __pydantic_parent_namespace__: Parent namespace of the model, used for automatic rebuilding of models.
  • __pydantic_post_init__: The name of the post-init method for the model, if defined.
  • __pydantic_root_model__: Whether the model is a RootModel.
  • __pydantic_serializer__: The pydantic-core SchemaSerializer used to dump instances of the model.
  • __pydantic_validator__: The pydantic-core SchemaValidator used to validate instances of the model.
  • __pydantic_extra__: An instance attribute with the values of extra fields from validation when model_config['extra'] == 'allow'.
  • __pydantic_fields_set__: An instance attribute with the names of fields explicitly set.
  • __pydantic_private__: Instance attribute with the values of private attributes set on the model instance.
Inherited Members
pydantic.main.BaseModel
BaseModel
model_extra
model_fields_set
model_construct
model_copy
model_dump
model_dump_json
model_json_schema
model_parametrized_name
model_rebuild
model_validate
model_validate_json
model_validate_strings
parse_file
from_orm
construct
schema
schema_json
validate
update_forward_refs
_ModelKind
model_kind_name
to_expression
sqlmesh.utils.pydantic.PydanticModel
dict
json
copy
parse_obj
parse_raw
missing_required_fields
extra_fields
all_fields
all_field_infos
required_fields
model_post_init
ModelKindMixin
is_symbolic
only_execution_time
full_history_restatement_only
class EmbeddedKind(_ModelKind):
570class EmbeddedKind(_ModelKind):
571    name: Literal[ModelKindName.EMBEDDED] = ModelKindName.EMBEDDED

Usage docs: https://docs.pydantic.dev/2.7/concepts/models/

A base class for creating Pydantic models.

Attributes:
  • __class_vars__: The names of classvars defined on the model.
  • __private_attributes__: Metadata about the private attributes of the model.
  • __signature__: The signature for instantiating the model.
  • __pydantic_complete__: Whether model building is completed, or if there are still undefined fields.
  • __pydantic_core_schema__: The pydantic-core schema used to build the SchemaValidator and SchemaSerializer.
  • __pydantic_custom_init__: Whether the model has a custom __init__ function.
  • __pydantic_decorators__: Metadata containing the decorators defined on the model. This replaces Model.__validators__ and Model.__root_validators__ from Pydantic V1.
  • __pydantic_generic_metadata__: Metadata for generic models; contains data used for a similar purpose to __args__, __origin__, __parameters__ in typing-module generics. May eventually be replaced by these.
  • __pydantic_parent_namespace__: Parent namespace of the model, used for automatic rebuilding of models.
  • __pydantic_post_init__: The name of the post-init method for the model, if defined.
  • __pydantic_root_model__: Whether the model is a RootModel.
  • __pydantic_serializer__: The pydantic-core SchemaSerializer used to dump instances of the model.
  • __pydantic_validator__: The pydantic-core SchemaValidator used to validate instances of the model.
  • __pydantic_extra__: An instance attribute with the values of extra fields from validation when model_config['extra'] == 'allow'.
  • __pydantic_fields_set__: An instance attribute with the names of fields explicitly set.
  • __pydantic_private__: Instance attribute with the values of private attributes set on the model instance.
Inherited Members
pydantic.main.BaseModel
BaseModel
model_extra
model_fields_set
model_construct
model_copy
model_dump
model_dump_json
model_json_schema
model_parametrized_name
model_rebuild
model_validate
model_validate_json
model_validate_strings
parse_file
from_orm
construct
schema
schema_json
validate
update_forward_refs
_ModelKind
model_kind_name
to_expression
sqlmesh.utils.pydantic.PydanticModel
dict
json
copy
parse_obj
parse_raw
missing_required_fields
extra_fields
all_fields
all_field_infos
required_fields
model_post_init
ModelKindMixin
is_symbolic
only_execution_time
full_history_restatement_only
class ExternalKind(_ModelKind):
574class ExternalKind(_ModelKind):
575    name: Literal[ModelKindName.EXTERNAL] = ModelKindName.EXTERNAL

Usage docs: https://docs.pydantic.dev/2.7/concepts/models/

A base class for creating Pydantic models.

Attributes:
  • __class_vars__: The names of classvars defined on the model.
  • __private_attributes__: Metadata about the private attributes of the model.
  • __signature__: The signature for instantiating the model.
  • __pydantic_complete__: Whether model building is completed, or if there are still undefined fields.
  • __pydantic_core_schema__: The pydantic-core schema used to build the SchemaValidator and SchemaSerializer.
  • __pydantic_custom_init__: Whether the model has a custom __init__ function.
  • __pydantic_decorators__: Metadata containing the decorators defined on the model. This replaces Model.__validators__ and Model.__root_validators__ from Pydantic V1.
  • __pydantic_generic_metadata__: Metadata for generic models; contains data used for a similar purpose to __args__, __origin__, __parameters__ in typing-module generics. May eventually be replaced by these.
  • __pydantic_parent_namespace__: Parent namespace of the model, used for automatic rebuilding of models.
  • __pydantic_post_init__: The name of the post-init method for the model, if defined.
  • __pydantic_root_model__: Whether the model is a RootModel.
  • __pydantic_serializer__: The pydantic-core SchemaSerializer used to dump instances of the model.
  • __pydantic_validator__: The pydantic-core SchemaValidator used to validate instances of the model.
  • __pydantic_extra__: An instance attribute with the values of extra fields from validation when model_config['extra'] == 'allow'.
  • __pydantic_fields_set__: An instance attribute with the names of fields explicitly set.
  • __pydantic_private__: Instance attribute with the values of private attributes set on the model instance.
Inherited Members
pydantic.main.BaseModel
BaseModel
model_extra
model_fields_set
model_construct
model_copy
model_dump
model_dump_json
model_json_schema
model_parametrized_name
model_rebuild
model_validate
model_validate_json
model_validate_strings
parse_file
from_orm
construct
schema
schema_json
validate
update_forward_refs
_ModelKind
model_kind_name
to_expression
sqlmesh.utils.pydantic.PydanticModel
dict
json
copy
parse_obj
parse_raw
missing_required_fields
extra_fields
all_fields
all_field_infos
required_fields
model_post_init
ModelKindMixin
is_symbolic
only_execution_time
full_history_restatement_only
609def model_kind_type_from_name(name: t.Optional[str]) -> t.Type[ModelKind]:
610    klass = MODEL_KIND_NAME_TO_TYPE.get(name) if name else None
611    if not klass:
612        raise ConfigError(f"Invalid model kind '{name}'")
613    return t.cast(t.Type[ModelKind], klass)
616def create_model_kind(v: t.Any, dialect: str, defaults: t.Dict[str, t.Any]) -> ModelKind:
617    if isinstance(v, _ModelKind):
618        return t.cast(ModelKind, v)
619
620    if isinstance(v, (d.ModelKind, dict)):
621        props = (
622            {prop.name: prop.args.get("value") for prop in v.expressions}
623            if isinstance(v, d.ModelKind)
624            else v
625        )
626        name = v.this if isinstance(v, d.ModelKind) else props.get("name")
627
628        # We want to ensure whatever name is provided to construct the class is the same name that will be
629        # found inside the class itself in order to avoid a change during plan/apply for legacy aliases.
630        # Ex: Pass in `SCD_TYPE_2` then we want to ensure we get `SCD_TYPE_2` as the kind name
631        # instead of `SCD_TYPE_2_BY_TIME`.
632        props["name"] = name
633        kind_type = model_kind_type_from_name(name)
634
635        if "dialect" in kind_type.all_fields() and props.get("dialect") is None:
636            props["dialect"] = dialect
637
638        # only pass the on_destructive_change user default to models inheriting from _Incremental
639        # that don't explicitly set it in the model definition
640        if (
641            issubclass(kind_type, _Incremental)
642            and props.get("on_destructive_change") is None
643            and defaults.get("on_destructive_change") is not None
644        ):
645            props["on_destructive_change"] = defaults.get("on_destructive_change")
646
647        return kind_type(**props)
648
649    name = (v.name if isinstance(v, exp.Expression) else str(v)).upper()
650    return model_kind_type_from_name(name)(name=name)  # type: ignore
def model_kind_validator(unknown):

Wrap a classmethod, staticmethod, property or unbound function and act as a descriptor that allows us to detect decorated items from the class' attributes.

This class' __get__ returns the wrapped item's __get__ result, which makes it transparent for classmethods and staticmethods.

Attributes:
  • wrapped: The decorator that has to be wrapped.
  • decorator_info: The decorator info.
  • shim: A wrapper function to wrap V1 style function.