sqlmesh.core.model.meta
1from __future__ import annotations 2 3import typing as t 4from enum import Enum 5from functools import cached_property 6from typing_extensions import Self 7 8from pydantic import Field 9from sqlglot import Dialect, exp, parse_one 10from sqlglot.helper import ensure_collection, ensure_list 11from sqlglot.optimizer.normalize_identifiers import normalize_identifiers 12 13from sqlmesh.core import dialect as d 14from sqlmesh.core.config.common import VirtualEnvironmentMode 15from sqlmesh.core.constants import LIQUID_CLUSTERING_KEYWORDS 16from sqlmesh.core.config.linter import LinterConfig 17from sqlmesh.core.dialect import normalize_model_name 18from sqlmesh.utils import classproperty 19from sqlmesh.core.model.common import ( 20 bool_validator, 21 default_catalog_validator, 22 depends_on_validator, 23 properties_validator, 24 parse_properties, 25) 26from sqlmesh.core.model.kind import ( 27 CustomKind, 28 IncrementalByUniqueKeyKind, 29 ModelKind, 30 OnDestructiveChange, 31 SCDType2ByColumnKind, 32 SCDType2ByTimeKind, 33 TimeColumn, 34 ViewKind, 35 model_kind_validator, 36 OnAdditiveChange, 37) 38from sqlmesh.core.node import _Node, str_or_exp_to_str 39from sqlmesh.core.reference import Reference 40from sqlmesh.utils.date import TimeLike 41from sqlmesh.utils.errors import ConfigError 42from sqlmesh.utils.pydantic import ( 43 ValidationInfo, 44 field_validator, 45 list_of_fields_validator, 46 model_validator, 47 get_dialect, 48 validation_data, 49) 50 51if t.TYPE_CHECKING: 52 from sqlmesh.core._typing import CustomMaterializationProperties, SessionProperties 53 from sqlmesh.core.engine_adapter._typing import GrantsConfig 54 55FunctionCall = t.Tuple[str, t.Dict[str, exp.Expr]] 56 57 58class GrantsTargetLayer(str, Enum): 59 """Target layer(s) where grants should be applied.""" 60 61 ALL = "all" 62 PHYSICAL = "physical" 63 VIRTUAL = "virtual" 64 65 @classproperty 66 def default(cls) -> "GrantsTargetLayer": 67 return GrantsTargetLayer.VIRTUAL 68 69 @property 70 def is_all(self) -> bool: 71 return self == GrantsTargetLayer.ALL 72 73 @property 74 def is_physical(self) -> bool: 75 return self == GrantsTargetLayer.PHYSICAL 76 77 @property 78 def is_virtual(self) -> bool: 79 return self == GrantsTargetLayer.VIRTUAL 80 81 def __str__(self) -> str: 82 return self.name 83 84 def __repr__(self) -> str: 85 return str(self) 86 87 88class ModelMeta(_Node): 89 """Metadata for models which can be defined in SQL.""" 90 91 dialect: str = "" 92 name: str 93 kind: ModelKind = ViewKind() 94 retention: t.Optional[int] = None # not implemented yet 95 table_format: t.Optional[str] = None 96 storage_format: t.Optional[str] = None 97 partitioned_by_: t.List[exp.Expr] = Field(default=[], alias="partitioned_by") 98 clustered_by: t.List[exp.Expr] = [] 99 default_catalog: t.Optional[str] = None 100 depends_on_: t.Optional[t.Set[str]] = Field(default=None, alias="depends_on") 101 columns_to_types_: t.Optional[t.Dict[str, exp.DataType]] = Field(default=None, alias="columns") 102 column_descriptions_: t.Optional[t.Dict[str, str]] = Field( 103 default=None, alias="column_descriptions" 104 ) 105 audits: t.List[FunctionCall] = [] 106 grains: t.List[exp.Expr] = [] 107 references: t.List[exp.Expr] = [] 108 physical_schema_override: t.Optional[str] = None 109 physical_properties_: t.Optional[exp.Tuple] = Field(default=None, alias="physical_properties") 110 virtual_properties_: t.Optional[exp.Tuple] = Field(default=None, alias="virtual_properties") 111 session_properties_: t.Optional[exp.Tuple] = Field(default=None, alias="session_properties") 112 allow_partials: bool = False 113 signals: t.List[FunctionCall] = [] 114 enabled: bool = True 115 physical_version: t.Optional[str] = None 116 gateway: t.Optional[str] = None 117 optimize_query: t.Optional[bool] = None 118 ignored_rules_: t.Optional[t.Set[str]] = Field( 119 default=None, exclude=True, alias="ignored_rules" 120 ) 121 formatting: t.Optional[bool] = Field(default=None, exclude=True) 122 virtual_environment_mode: VirtualEnvironmentMode = VirtualEnvironmentMode.default 123 grants_: t.Optional[exp.Tuple] = Field(default=None, alias="grants") 124 grants_target_layer: GrantsTargetLayer = GrantsTargetLayer.default 125 126 _bool_validator = bool_validator 127 _model_kind_validator = model_kind_validator 128 _properties_validator = properties_validator 129 _default_catalog_validator = default_catalog_validator 130 _depends_on_validator = depends_on_validator 131 132 @field_validator("audits", "signals", mode="before") 133 def _func_call_validator(cls, v: t.Any, field: t.Any) -> t.Any: 134 is_signal = getattr(field, "name" if hasattr(field, "name") else "field_name") == "signals" 135 136 return d.extract_function_calls(v, allow_tuples=is_signal) 137 138 @field_validator("tags", mode="before") 139 def _value_or_tuple_validator(cls, v: t.Any, info: ValidationInfo) -> t.Any: 140 return ensure_list(cls._validate_value_or_tuple(v, validation_data(info))) 141 142 @classmethod 143 def _validate_value_or_tuple( 144 cls, v: t.Dict[str, t.Any], data: t.Dict[str, t.Any], normalize: bool = False 145 ) -> t.Any: 146 dialect = data.get("dialect") 147 148 def _normalize(value: t.Any) -> t.Any: 149 return normalize_identifiers(value, dialect=dialect) if normalize else value 150 151 if isinstance(v, exp.Paren): 152 v = [v.unnest()] 153 154 if isinstance(v, (exp.Tuple, exp.Array)): 155 return [_normalize(e).name for e in v.expressions] 156 if isinstance(v, exp.Expr): 157 return _normalize(v).name 158 if isinstance(v, str): 159 value = _normalize(v) 160 return value.name if isinstance(value, exp.Expr) else value 161 if isinstance(v, (list, tuple)): 162 return [cls._validate_value_or_tuple(elm, data, normalize=normalize) for elm in v] 163 164 return v 165 166 @field_validator("table_format", "storage_format", mode="before") 167 def _format_validator(cls, v: t.Any, info: ValidationInfo) -> t.Optional[str]: 168 if isinstance(v, exp.Expr) and not (isinstance(v, (exp.Literal, exp.Identifier))): 169 return v.sql(validation_data(info).get("dialect")) 170 return str_or_exp_to_str(v) 171 172 @field_validator("dialect", mode="before") 173 def _dialect_validator(cls, v: t.Any) -> t.Optional[str]: 174 # dialects are parsed as identifiers and may get normalized as uppercase, 175 # so this ensures they'll be stored as lowercase 176 dialect = str_or_exp_to_str(v) 177 return dialect and dialect.lower() 178 179 @field_validator("physical_version", mode="before") 180 def _physical_version_validator(cls, v: t.Any) -> t.Optional[str]: 181 if v is None: 182 return v 183 return str_or_exp_to_str(v) 184 185 @field_validator("gateway", mode="before") 186 def _gateway_validator(cls, v: t.Any) -> t.Optional[str]: 187 if v is None: 188 return None 189 gateway = str_or_exp_to_str(v) 190 return gateway and gateway.lower() 191 192 @field_validator("partitioned_by_", "clustered_by", mode="before") 193 def _partition_and_cluster_validator(cls, v: t.Any, info: ValidationInfo) -> t.List[exp.Expr]: 194 field = info.field_name or "" 195 dialect = (get_dialect(info) or "").lower() 196 197 if ( 198 isinstance(v, list) 199 and all(isinstance(i, str) for i in v) 200 and field == "partitioned_by_" 201 ): 202 # this branch gets hit when we are deserializing from json because `partitioned_by` is stored as a List[str] 203 # however, we should only invoke this if the list contains strings because this validator is also 204 # called by Python models which might pass a List[exp.Expression] 205 string_to_parse = ( 206 f"({','.join(v)})" # recreate the (a, b, c) part of "partitioned_by (a, b, c)" 207 ) 208 parsed = parse_one( 209 string_to_parse, into=exp.PartitionedByProperty, dialect=get_dialect(info) 210 ) 211 v = parsed.this.expressions if isinstance(parsed.this, exp.Schema) else v 212 213 if isinstance(v, str) and field == "clustered_by": 214 v = [v] 215 216 if isinstance(v, list) and field == "clustered_by" and dialect == "databricks": 217 # When deserializing from JSON, clustered_by is stored as List[str]. 218 # Restore keyword sentinels (AUTO/NONE) before list_of_fields_validator normalises 219 # them into quoted columns. 220 v = [ 221 exp.Var(this=item.upper()) 222 if isinstance(item, str) and item.upper() in LIQUID_CLUSTERING_KEYWORDS 223 else item 224 for item in v 225 ] 226 227 expressions = list_of_fields_validator(v, validation_data(info)) 228 229 for expression in expressions: 230 # AUTO and NONE are Databricks liquid clustering keywords, not column references. 231 # Only skip for clustered_by with the Databricks dialect — meaningless elsewhere. 232 if ( 233 field == "clustered_by" 234 and dialect == "databricks" 235 and isinstance(expression, exp.Var) 236 and expression.name.upper() in LIQUID_CLUSTERING_KEYWORDS 237 ): 238 continue 239 240 num_cols = len(list(expression.find_all(exp.Column))) 241 242 error_msg: t.Optional[str] = None 243 if num_cols == 0: 244 error_msg = "does not contain a column" 245 elif num_cols > 1: 246 error_msg = "contains multiple columns" 247 248 if error_msg: 249 raise ConfigError(f"Field '{expression}' {error_msg}") 250 251 return expressions 252 253 @field_validator( 254 "columns_to_types_", "derived_columns_to_types", mode="before", check_fields=False 255 ) 256 def _columns_validator( 257 cls, v: t.Any, info: ValidationInfo 258 ) -> t.Optional[t.Dict[str, exp.DataType]]: 259 columns_to_types = {} 260 dialect = validation_data(info).get("dialect") 261 262 if isinstance(v, exp.Schema): 263 for column in v.expressions: 264 expr = column.args.get("kind") 265 if not isinstance(expr, exp.DataType): 266 raise ConfigError(f"Missing data type for column '{column.name}'.") 267 268 expr.meta["dialect"] = dialect 269 columns_to_types[normalize_identifiers(column, dialect=dialect).name] = expr 270 271 return columns_to_types 272 273 if isinstance(v, dict): 274 dialect_obj = Dialect.get_or_raise(dialect) 275 udt = dialect_obj.SUPPORTS_USER_DEFINED_TYPES 276 for k, data_type in v.items(): 277 is_string_type = isinstance(data_type, str) 278 expr = exp.DataType.build(data_type, dialect=dialect, udt=udt) 279 # When deserializing from a string (e.g. JSON roundtrip), normalize the type 280 # through the dialect's type system so that aliases (e.g. INT in BigQuery, 281 # which is an alias for INT64/BIGINT) are resolved to their canonical form. 282 # This ensures stable data hash computation across serialization/deserialization 283 # roundtrips. We skip this for DataType objects passed directly (Python API) 284 # since those should be used as-is. 285 if ( 286 is_string_type 287 and dialect 288 and expr.this 289 not in ( 290 exp.DataType.Type.USERDEFINED, 291 exp.DataType.Type.UNKNOWN, 292 ) 293 ): 294 sql_repr = expr.sql(dialect=dialect) 295 try: 296 normalized = parse_one(sql_repr, read=dialect, into=exp.DataType) 297 if normalized is not None: 298 expr = normalized 299 except Exception: 300 pass 301 expr.meta["dialect"] = dialect 302 columns_to_types[normalize_identifiers(k, dialect=dialect).name] = expr 303 304 return columns_to_types 305 306 return v 307 308 @field_validator("column_descriptions_", mode="before") 309 def _column_descriptions_validator( 310 cls, vs: t.Any, info: ValidationInfo 311 ) -> t.Optional[t.Dict[str, str]]: 312 data = validation_data(info) 313 dialect = data.get("dialect") 314 315 if vs is None: 316 return None 317 318 if isinstance(vs, exp.Paren): 319 vs = vs.flatten() 320 321 if isinstance(vs, (exp.Tuple, exp.Array)): 322 vs = vs.expressions 323 324 raw_col_descriptions = ( 325 vs 326 if isinstance(vs, dict) 327 else {".".join([part.this for part in v.this.parts]): v.expression.name for v in vs} 328 ) 329 330 col_descriptions = { 331 normalize_identifiers(k, dialect=dialect).name: v 332 for k, v in raw_col_descriptions.items() 333 } 334 335 columns_to_types = data.get("columns_to_types_") 336 if columns_to_types: 337 from sqlmesh.core.console import get_console 338 339 console = get_console() 340 for column_name in list(col_descriptions): 341 if column_name not in columns_to_types: 342 console.log_warning( 343 f"In model '{data.get('name', '<unknown>')}', a description is provided for column '{column_name}' but it is not a column in the model." 344 ) 345 del col_descriptions[column_name] 346 347 return col_descriptions 348 349 @field_validator("grains", "references", mode="before") 350 def _refs_validator(cls, vs: t.Any, info: ValidationInfo) -> t.List[exp.Expr]: 351 dialect = validation_data(info).get("dialect") 352 353 if isinstance(vs, exp.Paren): 354 vs = vs.unnest() 355 356 if isinstance(vs, (exp.Tuple, exp.Array)): 357 vs = vs.expressions 358 else: 359 vs = [ 360 d.parse_one(v, dialect=dialect) if isinstance(v, str) else v 361 for v in ensure_collection(vs) 362 ] 363 364 refs = [] 365 366 for v in vs: 367 v = exp.column(v) if isinstance(v, exp.Identifier) else v 368 v.meta["dialect"] = dialect 369 refs.append(v) 370 371 return refs 372 373 @field_validator("ignored_rules_", mode="before") 374 def ignored_rules_validator(cls, vs: t.Any) -> t.Any: 375 return LinterConfig._validate_rules(vs) 376 377 @field_validator("grants_target_layer", mode="before") 378 def _grants_target_layer_validator(cls, v: t.Any) -> t.Any: 379 if isinstance(v, exp.Identifier): 380 return v.this 381 if isinstance(v, exp.Literal) and v.is_string: 382 return v.this 383 return v 384 385 @field_validator("session_properties_", mode="before") 386 def session_properties_validator(cls, v: t.Any, info: ValidationInfo) -> t.Any: 387 # use the generic properties validator to parse the session properties 388 parsed_session_properties = parse_properties(type(cls), v, info) 389 if not parsed_session_properties: 390 return parsed_session_properties 391 392 for eq in parsed_session_properties: 393 prop_name = eq.left.name 394 395 if prop_name == "query_label": 396 query_label = eq.right 397 if not isinstance( 398 query_label, (exp.Array, exp.Tuple, exp.Paren, d.MacroFunc, d.MacroVar) 399 ): 400 raise ConfigError( 401 "Invalid value for `session_properties.query_label`. Must be an array or tuple." 402 ) 403 404 label_tuples: t.List[exp.Expr] = ( 405 [query_label.unnest()] 406 if isinstance(query_label, exp.Paren) 407 else query_label.expressions 408 ) 409 410 for label_tuple in label_tuples: 411 if not ( 412 isinstance(label_tuple, exp.Tuple) 413 and len(label_tuple.expressions) == 2 414 and all(isinstance(label, exp.Literal) for label in label_tuple.expressions) 415 ): 416 raise ConfigError( 417 "Invalid entry in `session_properties.query_label`. Must be tuples of string literals with length 2." 418 ) 419 elif prop_name == "authorization": 420 authorization = eq.right 421 if not ( 422 isinstance(authorization, exp.Literal) and authorization.is_string 423 ) and not isinstance(authorization, (d.MacroFunc, d.MacroVar)): 424 raise ConfigError( 425 "Invalid value for `session_properties.authorization`. Must be a string literal." 426 ) 427 elif prop_name == "query_tags": 428 query_tags = eq.right 429 if isinstance(query_tags, (d.MacroFunc, d.MacroVar)): 430 continue 431 432 if not isinstance(query_tags, (exp.Map, exp.VarMap)): 433 raise ConfigError( 434 "Invalid value for `session_properties.query_tags`. Must be a map." 435 ) 436 437 keys = query_tags.args.get("keys") 438 values = query_tags.args.get("values") 439 if not isinstance(keys, exp.Array) or not isinstance(values, exp.Array): 440 raise ConfigError( 441 "Invalid value for `session_properties.query_tags`. Must be a map with array " 442 "keys and array values." 443 ) 444 445 for key, value in zip(keys.expressions, values.expressions): 446 if not isinstance(key, exp.Literal) or not key.is_string: 447 raise ConfigError( 448 "Invalid key in `session_properties.query_tags`. Keys must be string literals." 449 ) 450 451 if not ( 452 isinstance(value, exp.Null) 453 or (isinstance(value, exp.Literal) and value.is_string) 454 ): 455 raise ConfigError( 456 "Invalid value in `session_properties.query_tags`. Values must be string " 457 "literals or NULL." 458 ) 459 460 return parsed_session_properties 461 462 @model_validator(mode="before") 463 def _pre_root_validator(cls, data: t.Any) -> t.Any: 464 if not isinstance(data, dict): 465 return data 466 467 grain = data.pop("grain", None) 468 if grain: 469 grains = data.get("grains") 470 if grains: 471 raise ConfigError( 472 f"Cannot use argument 'grain' ({grain}) with 'grains' ({grains}), use only grains" 473 ) 474 data["grains"] = ensure_list(grain) 475 476 table_properties = data.pop("table_properties", None) 477 if table_properties: 478 if not isinstance(table_properties, str): 479 # Do not warn when deserializing from the state. 480 model_name = data["name"] 481 from sqlmesh.core.console import get_console 482 483 get_console().log_warning( 484 f"Model '{model_name}' is using the `table_properties` attribute which is deprecated. Please use `physical_properties` instead." 485 ) 486 physical_properties = data.get("physical_properties") 487 if physical_properties: 488 raise ConfigError( 489 f"Cannot use argument 'table_properties' ({table_properties}) with 'physical_properties' ({physical_properties}), use only physical_properties." 490 ) 491 492 data["physical_properties"] = table_properties 493 494 return data 495 496 @model_validator(mode="after") 497 def _root_validator(self) -> Self: 498 kind: t.Any = self.kind 499 500 for field in ("partitioned_by_", "clustered_by"): 501 if ( 502 getattr(self, field, None) 503 and not kind.is_materialized 504 and not (kind.is_view and kind.materialized) 505 ): 506 name = field[:-1] if field.endswith("_") else field 507 raise ValueError(f"{name} field cannot be set for {kind.name} models") 508 if kind.is_incremental_by_partition and not getattr(self, "partitioned_by_", None): 509 raise ValueError(f"partitioned_by field is required for {kind.name} models") 510 511 # needs to be in a mode=after model validator so that the field validators have run to convert from Expression -> str 512 if (storage_format := self.storage_format) and storage_format.lower() in { 513 "iceberg", 514 "hive", 515 "hudi", 516 "delta", 517 }: 518 from sqlmesh.core.console import get_console 519 520 get_console().log_warning( 521 f"Model {self.name} has `storage_format` set to a table format '{storage_format}' which is deprecated. Please use the `table_format` property instead." 522 ) 523 524 # Validate grants configuration for model kind support 525 if self.grants is not None and not kind.supports_grants: 526 raise ValueError(f"grants cannot be set for {kind.name} models") 527 528 return self 529 530 @property 531 def time_column(self) -> t.Optional[TimeColumn]: 532 """The time column for incremental models.""" 533 return getattr(self.kind, "time_column", None) 534 535 @property 536 def unique_key(self) -> t.List[exp.Expr]: 537 if isinstance( 538 self.kind, (SCDType2ByTimeKind, SCDType2ByColumnKind, IncrementalByUniqueKeyKind) 539 ): 540 return self.kind.unique_key 541 return [] 542 543 @property 544 def column_descriptions(self) -> t.Dict[str, str]: 545 """A dictionary of column names to annotation comments.""" 546 return self.column_descriptions_ or {} 547 548 @property 549 def lookback(self) -> int: 550 """The incremental lookback window.""" 551 return getattr(self.kind, "lookback", 0) or 0 552 553 def lookback_start(self, start: TimeLike) -> TimeLike: 554 if self.lookback == 0: 555 return start 556 557 for _ in range(self.lookback): 558 start = self.interval_unit.cron_prev(start) 559 return start 560 561 @property 562 def batch_size(self) -> t.Optional[int]: 563 """The maximal number of units in a single task for a backfill.""" 564 return getattr(self.kind, "batch_size", None) 565 566 @property 567 def batch_concurrency(self) -> t.Optional[int]: 568 """The maximal number of batches that can run concurrently for a backfill.""" 569 return getattr(self.kind, "batch_concurrency", None) 570 571 @cached_property 572 def physical_properties(self) -> t.Dict[str, exp.Expr]: 573 """A dictionary of properties that will be applied to the physical layer. It replaces table_properties which is deprecated.""" 574 if self.physical_properties_: 575 return {e.this.name: e.expression for e in self.physical_properties_.expressions} 576 return {} 577 578 @cached_property 579 def virtual_properties(self) -> t.Dict[str, exp.Expr]: 580 """A dictionary of properties that will be applied to the virtual layer.""" 581 if self.virtual_properties_: 582 return {e.this.name: e.expression for e in self.virtual_properties_.expressions} 583 return {} 584 585 @property 586 def session_properties(self) -> SessionProperties: 587 """A dictionary of session properties.""" 588 if not self.session_properties_: 589 return {} 590 591 return d.interpret_key_value_pairs(self.session_properties_) 592 593 @property 594 def custom_materialization_properties(self) -> CustomMaterializationProperties: 595 if isinstance(self.kind, CustomKind): 596 return self.kind.materialization_properties 597 return {} 598 599 @cached_property 600 def grants(self) -> t.Optional[GrantsConfig]: 601 """A dictionary of grants mapping permission names to lists of grantees.""" 602 603 if self.grants_ is None: 604 return None 605 606 if not self.grants_.expressions: 607 return {} 608 609 grants_dict = {} 610 for eq_expr in self.grants_.expressions: 611 try: 612 permission_name = self._validate_config_expression(eq_expr.left) 613 grantee_list = self._validate_nested_config_values(eq_expr.expression) 614 grants_dict[permission_name] = grantee_list 615 except ConfigError as e: 616 permission_name = ( 617 eq_expr.left.name if hasattr(eq_expr.left, "name") else str(eq_expr.left) 618 ) 619 raise ConfigError(f"Invalid grants configuration for '{permission_name}': {e}") 620 621 return grants_dict if grants_dict else None 622 623 @property 624 def all_references(self) -> t.List[Reference]: 625 """All references including grains.""" 626 return [Reference(model_name=self.name, expression=e, unique=True) for e in self.grains] + [ 627 Reference(model_name=self.name, expression=e, unique=True) for e in self.references 628 ] 629 630 @property 631 def on(self) -> t.List[str]: 632 """The grains to be used as join condition in table_diff.""" 633 634 on: t.List[str] = [] 635 for expr in [ref.expression for ref in self.all_references if ref.unique]: 636 if isinstance(expr, exp.Tuple): 637 on.extend([key.this.sql(dialect=self.dialect) for key in expr.expressions]) 638 else: 639 # Handle a single Column or Paren expression 640 on.append(expr.this.sql(dialect=self.dialect)) 641 642 return on 643 644 @property 645 def managed_columns(self) -> t.Dict[str, exp.DataType]: 646 return getattr(self.kind, "managed_columns", {}) 647 648 @property 649 def when_matched(self) -> t.Optional[exp.Whens]: 650 if isinstance(self.kind, IncrementalByUniqueKeyKind): 651 return self.kind.when_matched 652 return None 653 654 @property 655 def merge_filter(self) -> t.Optional[exp.Expr]: 656 if isinstance(self.kind, IncrementalByUniqueKeyKind): 657 return self.kind.merge_filter 658 return None 659 660 @property 661 def catalog(self) -> t.Optional[str]: 662 """Returns the catalog of a model.""" 663 return self.fully_qualified_table.catalog 664 665 @cached_property 666 def fully_qualified_table(self) -> exp.Table: 667 return exp.to_table(self.fqn) 668 669 @cached_property 670 def fqn(self) -> str: 671 return normalize_model_name( 672 self.name, default_catalog=self.default_catalog, dialect=self.dialect 673 ) 674 675 @property 676 def on_destructive_change(self) -> OnDestructiveChange: 677 return getattr(self.kind, "on_destructive_change", OnDestructiveChange.ALLOW) 678 679 @property 680 def on_additive_change(self) -> OnAdditiveChange: 681 """Return the model's additive change setting if it has one.""" 682 return getattr(self.kind, "on_additive_change", OnAdditiveChange.ALLOW) 683 684 @property 685 def ignored_rules(self) -> t.Set[str]: 686 return self.ignored_rules_ or set() 687 688 def _validate_config_expression(self, expr: exp.Expr) -> str: 689 if isinstance(expr, (d.MacroFunc, d.MacroVar)): 690 raise ConfigError(f"Unresolved macro: {expr.sql(dialect=self.dialect)}") 691 692 if isinstance(expr, exp.Null): 693 raise ConfigError("NULL value") 694 695 if isinstance(expr, exp.Literal): 696 return str(expr.this).strip() 697 if isinstance(expr, (exp.Column, exp.Identifier)): 698 return expr.name 699 return expr.sql(dialect=self.dialect).strip() 700 701 def _validate_nested_config_values(self, value_expr: exp.Expr) -> t.List[str]: 702 result = [] 703 704 def flatten_expr(expr: exp.Expr) -> None: 705 if isinstance(expr, exp.Array): 706 for elem in expr.expressions: 707 flatten_expr(elem) 708 elif isinstance(expr, (exp.Tuple, exp.Paren)): 709 expressions = [expr.unnest()] if isinstance(expr, exp.Paren) else expr.expressions 710 for elem in expressions: 711 flatten_expr(elem) 712 else: 713 result.append(self._validate_config_expression(expr)) 714 715 flatten_expr(value_expr) 716 return result
59class GrantsTargetLayer(str, Enum): 60 """Target layer(s) where grants should be applied.""" 61 62 ALL = "all" 63 PHYSICAL = "physical" 64 VIRTUAL = "virtual" 65 66 @classproperty 67 def default(cls) -> "GrantsTargetLayer": 68 return GrantsTargetLayer.VIRTUAL 69 70 @property 71 def is_all(self) -> bool: 72 return self == GrantsTargetLayer.ALL 73 74 @property 75 def is_physical(self) -> bool: 76 return self == GrantsTargetLayer.PHYSICAL 77 78 @property 79 def is_virtual(self) -> bool: 80 return self == GrantsTargetLayer.VIRTUAL 81 82 def __str__(self) -> str: 83 return self.name 84 85 def __repr__(self) -> str: 86 return str(self)
Target layer(s) where grants should be applied.
Inherited Members
- enum.Enum
- name
- value
- builtins.str
- encode
- replace
- split
- rsplit
- join
- capitalize
- casefold
- title
- center
- count
- expandtabs
- find
- partition
- index
- ljust
- lower
- lstrip
- rfind
- rindex
- rjust
- rstrip
- rpartition
- splitlines
- strip
- swapcase
- translate
- upper
- startswith
- endswith
- removeprefix
- removesuffix
- isascii
- islower
- isupper
- istitle
- isspace
- isdecimal
- isdigit
- isnumeric
- isalpha
- isalnum
- isidentifier
- isprintable
- zfill
- format
- format_map
- maketrans
89class ModelMeta(_Node): 90 """Metadata for models which can be defined in SQL.""" 91 92 dialect: str = "" 93 name: str 94 kind: ModelKind = ViewKind() 95 retention: t.Optional[int] = None # not implemented yet 96 table_format: t.Optional[str] = None 97 storage_format: t.Optional[str] = None 98 partitioned_by_: t.List[exp.Expr] = Field(default=[], alias="partitioned_by") 99 clustered_by: t.List[exp.Expr] = [] 100 default_catalog: t.Optional[str] = None 101 depends_on_: t.Optional[t.Set[str]] = Field(default=None, alias="depends_on") 102 columns_to_types_: t.Optional[t.Dict[str, exp.DataType]] = Field(default=None, alias="columns") 103 column_descriptions_: t.Optional[t.Dict[str, str]] = Field( 104 default=None, alias="column_descriptions" 105 ) 106 audits: t.List[FunctionCall] = [] 107 grains: t.List[exp.Expr] = [] 108 references: t.List[exp.Expr] = [] 109 physical_schema_override: t.Optional[str] = None 110 physical_properties_: t.Optional[exp.Tuple] = Field(default=None, alias="physical_properties") 111 virtual_properties_: t.Optional[exp.Tuple] = Field(default=None, alias="virtual_properties") 112 session_properties_: t.Optional[exp.Tuple] = Field(default=None, alias="session_properties") 113 allow_partials: bool = False 114 signals: t.List[FunctionCall] = [] 115 enabled: bool = True 116 physical_version: t.Optional[str] = None 117 gateway: t.Optional[str] = None 118 optimize_query: t.Optional[bool] = None 119 ignored_rules_: t.Optional[t.Set[str]] = Field( 120 default=None, exclude=True, alias="ignored_rules" 121 ) 122 formatting: t.Optional[bool] = Field(default=None, exclude=True) 123 virtual_environment_mode: VirtualEnvironmentMode = VirtualEnvironmentMode.default 124 grants_: t.Optional[exp.Tuple] = Field(default=None, alias="grants") 125 grants_target_layer: GrantsTargetLayer = GrantsTargetLayer.default 126 127 _bool_validator = bool_validator 128 _model_kind_validator = model_kind_validator 129 _properties_validator = properties_validator 130 _default_catalog_validator = default_catalog_validator 131 _depends_on_validator = depends_on_validator 132 133 @field_validator("audits", "signals", mode="before") 134 def _func_call_validator(cls, v: t.Any, field: t.Any) -> t.Any: 135 is_signal = getattr(field, "name" if hasattr(field, "name") else "field_name") == "signals" 136 137 return d.extract_function_calls(v, allow_tuples=is_signal) 138 139 @field_validator("tags", mode="before") 140 def _value_or_tuple_validator(cls, v: t.Any, info: ValidationInfo) -> t.Any: 141 return ensure_list(cls._validate_value_or_tuple(v, validation_data(info))) 142 143 @classmethod 144 def _validate_value_or_tuple( 145 cls, v: t.Dict[str, t.Any], data: t.Dict[str, t.Any], normalize: bool = False 146 ) -> t.Any: 147 dialect = data.get("dialect") 148 149 def _normalize(value: t.Any) -> t.Any: 150 return normalize_identifiers(value, dialect=dialect) if normalize else value 151 152 if isinstance(v, exp.Paren): 153 v = [v.unnest()] 154 155 if isinstance(v, (exp.Tuple, exp.Array)): 156 return [_normalize(e).name for e in v.expressions] 157 if isinstance(v, exp.Expr): 158 return _normalize(v).name 159 if isinstance(v, str): 160 value = _normalize(v) 161 return value.name if isinstance(value, exp.Expr) else value 162 if isinstance(v, (list, tuple)): 163 return [cls._validate_value_or_tuple(elm, data, normalize=normalize) for elm in v] 164 165 return v 166 167 @field_validator("table_format", "storage_format", mode="before") 168 def _format_validator(cls, v: t.Any, info: ValidationInfo) -> t.Optional[str]: 169 if isinstance(v, exp.Expr) and not (isinstance(v, (exp.Literal, exp.Identifier))): 170 return v.sql(validation_data(info).get("dialect")) 171 return str_or_exp_to_str(v) 172 173 @field_validator("dialect", mode="before") 174 def _dialect_validator(cls, v: t.Any) -> t.Optional[str]: 175 # dialects are parsed as identifiers and may get normalized as uppercase, 176 # so this ensures they'll be stored as lowercase 177 dialect = str_or_exp_to_str(v) 178 return dialect and dialect.lower() 179 180 @field_validator("physical_version", mode="before") 181 def _physical_version_validator(cls, v: t.Any) -> t.Optional[str]: 182 if v is None: 183 return v 184 return str_or_exp_to_str(v) 185 186 @field_validator("gateway", mode="before") 187 def _gateway_validator(cls, v: t.Any) -> t.Optional[str]: 188 if v is None: 189 return None 190 gateway = str_or_exp_to_str(v) 191 return gateway and gateway.lower() 192 193 @field_validator("partitioned_by_", "clustered_by", mode="before") 194 def _partition_and_cluster_validator(cls, v: t.Any, info: ValidationInfo) -> t.List[exp.Expr]: 195 field = info.field_name or "" 196 dialect = (get_dialect(info) or "").lower() 197 198 if ( 199 isinstance(v, list) 200 and all(isinstance(i, str) for i in v) 201 and field == "partitioned_by_" 202 ): 203 # this branch gets hit when we are deserializing from json because `partitioned_by` is stored as a List[str] 204 # however, we should only invoke this if the list contains strings because this validator is also 205 # called by Python models which might pass a List[exp.Expression] 206 string_to_parse = ( 207 f"({','.join(v)})" # recreate the (a, b, c) part of "partitioned_by (a, b, c)" 208 ) 209 parsed = parse_one( 210 string_to_parse, into=exp.PartitionedByProperty, dialect=get_dialect(info) 211 ) 212 v = parsed.this.expressions if isinstance(parsed.this, exp.Schema) else v 213 214 if isinstance(v, str) and field == "clustered_by": 215 v = [v] 216 217 if isinstance(v, list) and field == "clustered_by" and dialect == "databricks": 218 # When deserializing from JSON, clustered_by is stored as List[str]. 219 # Restore keyword sentinels (AUTO/NONE) before list_of_fields_validator normalises 220 # them into quoted columns. 221 v = [ 222 exp.Var(this=item.upper()) 223 if isinstance(item, str) and item.upper() in LIQUID_CLUSTERING_KEYWORDS 224 else item 225 for item in v 226 ] 227 228 expressions = list_of_fields_validator(v, validation_data(info)) 229 230 for expression in expressions: 231 # AUTO and NONE are Databricks liquid clustering keywords, not column references. 232 # Only skip for clustered_by with the Databricks dialect — meaningless elsewhere. 233 if ( 234 field == "clustered_by" 235 and dialect == "databricks" 236 and isinstance(expression, exp.Var) 237 and expression.name.upper() in LIQUID_CLUSTERING_KEYWORDS 238 ): 239 continue 240 241 num_cols = len(list(expression.find_all(exp.Column))) 242 243 error_msg: t.Optional[str] = None 244 if num_cols == 0: 245 error_msg = "does not contain a column" 246 elif num_cols > 1: 247 error_msg = "contains multiple columns" 248 249 if error_msg: 250 raise ConfigError(f"Field '{expression}' {error_msg}") 251 252 return expressions 253 254 @field_validator( 255 "columns_to_types_", "derived_columns_to_types", mode="before", check_fields=False 256 ) 257 def _columns_validator( 258 cls, v: t.Any, info: ValidationInfo 259 ) -> t.Optional[t.Dict[str, exp.DataType]]: 260 columns_to_types = {} 261 dialect = validation_data(info).get("dialect") 262 263 if isinstance(v, exp.Schema): 264 for column in v.expressions: 265 expr = column.args.get("kind") 266 if not isinstance(expr, exp.DataType): 267 raise ConfigError(f"Missing data type for column '{column.name}'.") 268 269 expr.meta["dialect"] = dialect 270 columns_to_types[normalize_identifiers(column, dialect=dialect).name] = expr 271 272 return columns_to_types 273 274 if isinstance(v, dict): 275 dialect_obj = Dialect.get_or_raise(dialect) 276 udt = dialect_obj.SUPPORTS_USER_DEFINED_TYPES 277 for k, data_type in v.items(): 278 is_string_type = isinstance(data_type, str) 279 expr = exp.DataType.build(data_type, dialect=dialect, udt=udt) 280 # When deserializing from a string (e.g. JSON roundtrip), normalize the type 281 # through the dialect's type system so that aliases (e.g. INT in BigQuery, 282 # which is an alias for INT64/BIGINT) are resolved to their canonical form. 283 # This ensures stable data hash computation across serialization/deserialization 284 # roundtrips. We skip this for DataType objects passed directly (Python API) 285 # since those should be used as-is. 286 if ( 287 is_string_type 288 and dialect 289 and expr.this 290 not in ( 291 exp.DataType.Type.USERDEFINED, 292 exp.DataType.Type.UNKNOWN, 293 ) 294 ): 295 sql_repr = expr.sql(dialect=dialect) 296 try: 297 normalized = parse_one(sql_repr, read=dialect, into=exp.DataType) 298 if normalized is not None: 299 expr = normalized 300 except Exception: 301 pass 302 expr.meta["dialect"] = dialect 303 columns_to_types[normalize_identifiers(k, dialect=dialect).name] = expr 304 305 return columns_to_types 306 307 return v 308 309 @field_validator("column_descriptions_", mode="before") 310 def _column_descriptions_validator( 311 cls, vs: t.Any, info: ValidationInfo 312 ) -> t.Optional[t.Dict[str, str]]: 313 data = validation_data(info) 314 dialect = data.get("dialect") 315 316 if vs is None: 317 return None 318 319 if isinstance(vs, exp.Paren): 320 vs = vs.flatten() 321 322 if isinstance(vs, (exp.Tuple, exp.Array)): 323 vs = vs.expressions 324 325 raw_col_descriptions = ( 326 vs 327 if isinstance(vs, dict) 328 else {".".join([part.this for part in v.this.parts]): v.expression.name for v in vs} 329 ) 330 331 col_descriptions = { 332 normalize_identifiers(k, dialect=dialect).name: v 333 for k, v in raw_col_descriptions.items() 334 } 335 336 columns_to_types = data.get("columns_to_types_") 337 if columns_to_types: 338 from sqlmesh.core.console import get_console 339 340 console = get_console() 341 for column_name in list(col_descriptions): 342 if column_name not in columns_to_types: 343 console.log_warning( 344 f"In model '{data.get('name', '<unknown>')}', a description is provided for column '{column_name}' but it is not a column in the model." 345 ) 346 del col_descriptions[column_name] 347 348 return col_descriptions 349 350 @field_validator("grains", "references", mode="before") 351 def _refs_validator(cls, vs: t.Any, info: ValidationInfo) -> t.List[exp.Expr]: 352 dialect = validation_data(info).get("dialect") 353 354 if isinstance(vs, exp.Paren): 355 vs = vs.unnest() 356 357 if isinstance(vs, (exp.Tuple, exp.Array)): 358 vs = vs.expressions 359 else: 360 vs = [ 361 d.parse_one(v, dialect=dialect) if isinstance(v, str) else v 362 for v in ensure_collection(vs) 363 ] 364 365 refs = [] 366 367 for v in vs: 368 v = exp.column(v) if isinstance(v, exp.Identifier) else v 369 v.meta["dialect"] = dialect 370 refs.append(v) 371 372 return refs 373 374 @field_validator("ignored_rules_", mode="before") 375 def ignored_rules_validator(cls, vs: t.Any) -> t.Any: 376 return LinterConfig._validate_rules(vs) 377 378 @field_validator("grants_target_layer", mode="before") 379 def _grants_target_layer_validator(cls, v: t.Any) -> t.Any: 380 if isinstance(v, exp.Identifier): 381 return v.this 382 if isinstance(v, exp.Literal) and v.is_string: 383 return v.this 384 return v 385 386 @field_validator("session_properties_", mode="before") 387 def session_properties_validator(cls, v: t.Any, info: ValidationInfo) -> t.Any: 388 # use the generic properties validator to parse the session properties 389 parsed_session_properties = parse_properties(type(cls), v, info) 390 if not parsed_session_properties: 391 return parsed_session_properties 392 393 for eq in parsed_session_properties: 394 prop_name = eq.left.name 395 396 if prop_name == "query_label": 397 query_label = eq.right 398 if not isinstance( 399 query_label, (exp.Array, exp.Tuple, exp.Paren, d.MacroFunc, d.MacroVar) 400 ): 401 raise ConfigError( 402 "Invalid value for `session_properties.query_label`. Must be an array or tuple." 403 ) 404 405 label_tuples: t.List[exp.Expr] = ( 406 [query_label.unnest()] 407 if isinstance(query_label, exp.Paren) 408 else query_label.expressions 409 ) 410 411 for label_tuple in label_tuples: 412 if not ( 413 isinstance(label_tuple, exp.Tuple) 414 and len(label_tuple.expressions) == 2 415 and all(isinstance(label, exp.Literal) for label in label_tuple.expressions) 416 ): 417 raise ConfigError( 418 "Invalid entry in `session_properties.query_label`. Must be tuples of string literals with length 2." 419 ) 420 elif prop_name == "authorization": 421 authorization = eq.right 422 if not ( 423 isinstance(authorization, exp.Literal) and authorization.is_string 424 ) and not isinstance(authorization, (d.MacroFunc, d.MacroVar)): 425 raise ConfigError( 426 "Invalid value for `session_properties.authorization`. Must be a string literal." 427 ) 428 elif prop_name == "query_tags": 429 query_tags = eq.right 430 if isinstance(query_tags, (d.MacroFunc, d.MacroVar)): 431 continue 432 433 if not isinstance(query_tags, (exp.Map, exp.VarMap)): 434 raise ConfigError( 435 "Invalid value for `session_properties.query_tags`. Must be a map." 436 ) 437 438 keys = query_tags.args.get("keys") 439 values = query_tags.args.get("values") 440 if not isinstance(keys, exp.Array) or not isinstance(values, exp.Array): 441 raise ConfigError( 442 "Invalid value for `session_properties.query_tags`. Must be a map with array " 443 "keys and array values." 444 ) 445 446 for key, value in zip(keys.expressions, values.expressions): 447 if not isinstance(key, exp.Literal) or not key.is_string: 448 raise ConfigError( 449 "Invalid key in `session_properties.query_tags`. Keys must be string literals." 450 ) 451 452 if not ( 453 isinstance(value, exp.Null) 454 or (isinstance(value, exp.Literal) and value.is_string) 455 ): 456 raise ConfigError( 457 "Invalid value in `session_properties.query_tags`. Values must be string " 458 "literals or NULL." 459 ) 460 461 return parsed_session_properties 462 463 @model_validator(mode="before") 464 def _pre_root_validator(cls, data: t.Any) -> t.Any: 465 if not isinstance(data, dict): 466 return data 467 468 grain = data.pop("grain", None) 469 if grain: 470 grains = data.get("grains") 471 if grains: 472 raise ConfigError( 473 f"Cannot use argument 'grain' ({grain}) with 'grains' ({grains}), use only grains" 474 ) 475 data["grains"] = ensure_list(grain) 476 477 table_properties = data.pop("table_properties", None) 478 if table_properties: 479 if not isinstance(table_properties, str): 480 # Do not warn when deserializing from the state. 481 model_name = data["name"] 482 from sqlmesh.core.console import get_console 483 484 get_console().log_warning( 485 f"Model '{model_name}' is using the `table_properties` attribute which is deprecated. Please use `physical_properties` instead." 486 ) 487 physical_properties = data.get("physical_properties") 488 if physical_properties: 489 raise ConfigError( 490 f"Cannot use argument 'table_properties' ({table_properties}) with 'physical_properties' ({physical_properties}), use only physical_properties." 491 ) 492 493 data["physical_properties"] = table_properties 494 495 return data 496 497 @model_validator(mode="after") 498 def _root_validator(self) -> Self: 499 kind: t.Any = self.kind 500 501 for field in ("partitioned_by_", "clustered_by"): 502 if ( 503 getattr(self, field, None) 504 and not kind.is_materialized 505 and not (kind.is_view and kind.materialized) 506 ): 507 name = field[:-1] if field.endswith("_") else field 508 raise ValueError(f"{name} field cannot be set for {kind.name} models") 509 if kind.is_incremental_by_partition and not getattr(self, "partitioned_by_", None): 510 raise ValueError(f"partitioned_by field is required for {kind.name} models") 511 512 # needs to be in a mode=after model validator so that the field validators have run to convert from Expression -> str 513 if (storage_format := self.storage_format) and storage_format.lower() in { 514 "iceberg", 515 "hive", 516 "hudi", 517 "delta", 518 }: 519 from sqlmesh.core.console import get_console 520 521 get_console().log_warning( 522 f"Model {self.name} has `storage_format` set to a table format '{storage_format}' which is deprecated. Please use the `table_format` property instead." 523 ) 524 525 # Validate grants configuration for model kind support 526 if self.grants is not None and not kind.supports_grants: 527 raise ValueError(f"grants cannot be set for {kind.name} models") 528 529 return self 530 531 @property 532 def time_column(self) -> t.Optional[TimeColumn]: 533 """The time column for incremental models.""" 534 return getattr(self.kind, "time_column", None) 535 536 @property 537 def unique_key(self) -> t.List[exp.Expr]: 538 if isinstance( 539 self.kind, (SCDType2ByTimeKind, SCDType2ByColumnKind, IncrementalByUniqueKeyKind) 540 ): 541 return self.kind.unique_key 542 return [] 543 544 @property 545 def column_descriptions(self) -> t.Dict[str, str]: 546 """A dictionary of column names to annotation comments.""" 547 return self.column_descriptions_ or {} 548 549 @property 550 def lookback(self) -> int: 551 """The incremental lookback window.""" 552 return getattr(self.kind, "lookback", 0) or 0 553 554 def lookback_start(self, start: TimeLike) -> TimeLike: 555 if self.lookback == 0: 556 return start 557 558 for _ in range(self.lookback): 559 start = self.interval_unit.cron_prev(start) 560 return start 561 562 @property 563 def batch_size(self) -> t.Optional[int]: 564 """The maximal number of units in a single task for a backfill.""" 565 return getattr(self.kind, "batch_size", None) 566 567 @property 568 def batch_concurrency(self) -> t.Optional[int]: 569 """The maximal number of batches that can run concurrently for a backfill.""" 570 return getattr(self.kind, "batch_concurrency", None) 571 572 @cached_property 573 def physical_properties(self) -> t.Dict[str, exp.Expr]: 574 """A dictionary of properties that will be applied to the physical layer. It replaces table_properties which is deprecated.""" 575 if self.physical_properties_: 576 return {e.this.name: e.expression for e in self.physical_properties_.expressions} 577 return {} 578 579 @cached_property 580 def virtual_properties(self) -> t.Dict[str, exp.Expr]: 581 """A dictionary of properties that will be applied to the virtual layer.""" 582 if self.virtual_properties_: 583 return {e.this.name: e.expression for e in self.virtual_properties_.expressions} 584 return {} 585 586 @property 587 def session_properties(self) -> SessionProperties: 588 """A dictionary of session properties.""" 589 if not self.session_properties_: 590 return {} 591 592 return d.interpret_key_value_pairs(self.session_properties_) 593 594 @property 595 def custom_materialization_properties(self) -> CustomMaterializationProperties: 596 if isinstance(self.kind, CustomKind): 597 return self.kind.materialization_properties 598 return {} 599 600 @cached_property 601 def grants(self) -> t.Optional[GrantsConfig]: 602 """A dictionary of grants mapping permission names to lists of grantees.""" 603 604 if self.grants_ is None: 605 return None 606 607 if not self.grants_.expressions: 608 return {} 609 610 grants_dict = {} 611 for eq_expr in self.grants_.expressions: 612 try: 613 permission_name = self._validate_config_expression(eq_expr.left) 614 grantee_list = self._validate_nested_config_values(eq_expr.expression) 615 grants_dict[permission_name] = grantee_list 616 except ConfigError as e: 617 permission_name = ( 618 eq_expr.left.name if hasattr(eq_expr.left, "name") else str(eq_expr.left) 619 ) 620 raise ConfigError(f"Invalid grants configuration for '{permission_name}': {e}") 621 622 return grants_dict if grants_dict else None 623 624 @property 625 def all_references(self) -> t.List[Reference]: 626 """All references including grains.""" 627 return [Reference(model_name=self.name, expression=e, unique=True) for e in self.grains] + [ 628 Reference(model_name=self.name, expression=e, unique=True) for e in self.references 629 ] 630 631 @property 632 def on(self) -> t.List[str]: 633 """The grains to be used as join condition in table_diff.""" 634 635 on: t.List[str] = [] 636 for expr in [ref.expression for ref in self.all_references if ref.unique]: 637 if isinstance(expr, exp.Tuple): 638 on.extend([key.this.sql(dialect=self.dialect) for key in expr.expressions]) 639 else: 640 # Handle a single Column or Paren expression 641 on.append(expr.this.sql(dialect=self.dialect)) 642 643 return on 644 645 @property 646 def managed_columns(self) -> t.Dict[str, exp.DataType]: 647 return getattr(self.kind, "managed_columns", {}) 648 649 @property 650 def when_matched(self) -> t.Optional[exp.Whens]: 651 if isinstance(self.kind, IncrementalByUniqueKeyKind): 652 return self.kind.when_matched 653 return None 654 655 @property 656 def merge_filter(self) -> t.Optional[exp.Expr]: 657 if isinstance(self.kind, IncrementalByUniqueKeyKind): 658 return self.kind.merge_filter 659 return None 660 661 @property 662 def catalog(self) -> t.Optional[str]: 663 """Returns the catalog of a model.""" 664 return self.fully_qualified_table.catalog 665 666 @cached_property 667 def fully_qualified_table(self) -> exp.Table: 668 return exp.to_table(self.fqn) 669 670 @cached_property 671 def fqn(self) -> str: 672 return normalize_model_name( 673 self.name, default_catalog=self.default_catalog, dialect=self.dialect 674 ) 675 676 @property 677 def on_destructive_change(self) -> OnDestructiveChange: 678 return getattr(self.kind, "on_destructive_change", OnDestructiveChange.ALLOW) 679 680 @property 681 def on_additive_change(self) -> OnAdditiveChange: 682 """Return the model's additive change setting if it has one.""" 683 return getattr(self.kind, "on_additive_change", OnAdditiveChange.ALLOW) 684 685 @property 686 def ignored_rules(self) -> t.Set[str]: 687 return self.ignored_rules_ or set() 688 689 def _validate_config_expression(self, expr: exp.Expr) -> str: 690 if isinstance(expr, (d.MacroFunc, d.MacroVar)): 691 raise ConfigError(f"Unresolved macro: {expr.sql(dialect=self.dialect)}") 692 693 if isinstance(expr, exp.Null): 694 raise ConfigError("NULL value") 695 696 if isinstance(expr, exp.Literal): 697 return str(expr.this).strip() 698 if isinstance(expr, (exp.Column, exp.Identifier)): 699 return expr.name 700 return expr.sql(dialect=self.dialect).strip() 701 702 def _validate_nested_config_values(self, value_expr: exp.Expr) -> t.List[str]: 703 result = [] 704 705 def flatten_expr(expr: exp.Expr) -> None: 706 if isinstance(expr, exp.Array): 707 for elem in expr.expressions: 708 flatten_expr(elem) 709 elif isinstance(expr, (exp.Tuple, exp.Paren)): 710 expressions = [expr.unnest()] if isinstance(expr, exp.Paren) else expr.expressions 711 for elem in expressions: 712 flatten_expr(elem) 713 else: 714 result.append(self._validate_config_expression(expr)) 715 716 flatten_expr(value_expr) 717 return result
Metadata for models which can be defined in SQL.
386 @field_validator("session_properties_", mode="before") 387 def session_properties_validator(cls, v: t.Any, info: ValidationInfo) -> t.Any: 388 # use the generic properties validator to parse the session properties 389 parsed_session_properties = parse_properties(type(cls), v, info) 390 if not parsed_session_properties: 391 return parsed_session_properties 392 393 for eq in parsed_session_properties: 394 prop_name = eq.left.name 395 396 if prop_name == "query_label": 397 query_label = eq.right 398 if not isinstance( 399 query_label, (exp.Array, exp.Tuple, exp.Paren, d.MacroFunc, d.MacroVar) 400 ): 401 raise ConfigError( 402 "Invalid value for `session_properties.query_label`. Must be an array or tuple." 403 ) 404 405 label_tuples: t.List[exp.Expr] = ( 406 [query_label.unnest()] 407 if isinstance(query_label, exp.Paren) 408 else query_label.expressions 409 ) 410 411 for label_tuple in label_tuples: 412 if not ( 413 isinstance(label_tuple, exp.Tuple) 414 and len(label_tuple.expressions) == 2 415 and all(isinstance(label, exp.Literal) for label in label_tuple.expressions) 416 ): 417 raise ConfigError( 418 "Invalid entry in `session_properties.query_label`. Must be tuples of string literals with length 2." 419 ) 420 elif prop_name == "authorization": 421 authorization = eq.right 422 if not ( 423 isinstance(authorization, exp.Literal) and authorization.is_string 424 ) and not isinstance(authorization, (d.MacroFunc, d.MacroVar)): 425 raise ConfigError( 426 "Invalid value for `session_properties.authorization`. Must be a string literal." 427 ) 428 elif prop_name == "query_tags": 429 query_tags = eq.right 430 if isinstance(query_tags, (d.MacroFunc, d.MacroVar)): 431 continue 432 433 if not isinstance(query_tags, (exp.Map, exp.VarMap)): 434 raise ConfigError( 435 "Invalid value for `session_properties.query_tags`. Must be a map." 436 ) 437 438 keys = query_tags.args.get("keys") 439 values = query_tags.args.get("values") 440 if not isinstance(keys, exp.Array) or not isinstance(values, exp.Array): 441 raise ConfigError( 442 "Invalid value for `session_properties.query_tags`. Must be a map with array " 443 "keys and array values." 444 ) 445 446 for key, value in zip(keys.expressions, values.expressions): 447 if not isinstance(key, exp.Literal) or not key.is_string: 448 raise ConfigError( 449 "Invalid key in `session_properties.query_tags`. Keys must be string literals." 450 ) 451 452 if not ( 453 isinstance(value, exp.Null) 454 or (isinstance(value, exp.Literal) and value.is_string) 455 ): 456 raise ConfigError( 457 "Invalid value in `session_properties.query_tags`. Values must be string " 458 "literals or NULL." 459 ) 460 461 return parsed_session_properties
531 @property 532 def time_column(self) -> t.Optional[TimeColumn]: 533 """The time column for incremental models.""" 534 return getattr(self.kind, "time_column", None)
The time column for incremental models.
544 @property 545 def column_descriptions(self) -> t.Dict[str, str]: 546 """A dictionary of column names to annotation comments.""" 547 return self.column_descriptions_ or {}
A dictionary of column names to annotation comments.
549 @property 550 def lookback(self) -> int: 551 """The incremental lookback window.""" 552 return getattr(self.kind, "lookback", 0) or 0
The incremental lookback window.
562 @property 563 def batch_size(self) -> t.Optional[int]: 564 """The maximal number of units in a single task for a backfill.""" 565 return getattr(self.kind, "batch_size", None)
The maximal number of units in a single task for a backfill.
567 @property 568 def batch_concurrency(self) -> t.Optional[int]: 569 """The maximal number of batches that can run concurrently for a backfill.""" 570 return getattr(self.kind, "batch_concurrency", None)
The maximal number of batches that can run concurrently for a backfill.
572 @cached_property 573 def physical_properties(self) -> t.Dict[str, exp.Expr]: 574 """A dictionary of properties that will be applied to the physical layer. It replaces table_properties which is deprecated.""" 575 if self.physical_properties_: 576 return {e.this.name: e.expression for e in self.physical_properties_.expressions} 577 return {}
A dictionary of properties that will be applied to the physical layer. It replaces table_properties which is deprecated.
579 @cached_property 580 def virtual_properties(self) -> t.Dict[str, exp.Expr]: 581 """A dictionary of properties that will be applied to the virtual layer.""" 582 if self.virtual_properties_: 583 return {e.this.name: e.expression for e in self.virtual_properties_.expressions} 584 return {}
A dictionary of properties that will be applied to the virtual layer.
586 @property 587 def session_properties(self) -> SessionProperties: 588 """A dictionary of session properties.""" 589 if not self.session_properties_: 590 return {} 591 592 return d.interpret_key_value_pairs(self.session_properties_)
A dictionary of session properties.
600 @cached_property 601 def grants(self) -> t.Optional[GrantsConfig]: 602 """A dictionary of grants mapping permission names to lists of grantees.""" 603 604 if self.grants_ is None: 605 return None 606 607 if not self.grants_.expressions: 608 return {} 609 610 grants_dict = {} 611 for eq_expr in self.grants_.expressions: 612 try: 613 permission_name = self._validate_config_expression(eq_expr.left) 614 grantee_list = self._validate_nested_config_values(eq_expr.expression) 615 grants_dict[permission_name] = grantee_list 616 except ConfigError as e: 617 permission_name = ( 618 eq_expr.left.name if hasattr(eq_expr.left, "name") else str(eq_expr.left) 619 ) 620 raise ConfigError(f"Invalid grants configuration for '{permission_name}': {e}") 621 622 return grants_dict if grants_dict else None
A dictionary of grants mapping permission names to lists of grantees.
624 @property 625 def all_references(self) -> t.List[Reference]: 626 """All references including grains.""" 627 return [Reference(model_name=self.name, expression=e, unique=True) for e in self.grains] + [ 628 Reference(model_name=self.name, expression=e, unique=True) for e in self.references 629 ]
All references including grains.
631 @property 632 def on(self) -> t.List[str]: 633 """The grains to be used as join condition in table_diff.""" 634 635 on: t.List[str] = [] 636 for expr in [ref.expression for ref in self.all_references if ref.unique]: 637 if isinstance(expr, exp.Tuple): 638 on.extend([key.this.sql(dialect=self.dialect) for key in expr.expressions]) 639 else: 640 # Handle a single Column or Paren expression 641 on.append(expr.this.sql(dialect=self.dialect)) 642 643 return on
The grains to be used as join condition in table_diff.
661 @property 662 def catalog(self) -> t.Optional[str]: 663 """Returns the catalog of a model.""" 664 return self.fully_qualified_table.catalog
Returns the catalog of a model.
680 @property 681 def on_additive_change(self) -> OnAdditiveChange: 682 """Return the model's additive change setting if it has one.""" 683 return getattr(self.kind, "on_additive_change", OnAdditiveChange.ALLOW)
Return the model's additive change setting if it has one.
Configuration for the model, should be a dictionary conforming to [ConfigDict][pydantic.config.ConfigDict].
365def init_private_attributes(self: BaseModel, context: Any, /) -> None: 366 """This function is meant to behave like a BaseModel method to initialize private attributes. 367 368 It takes context as an argument since that's what pydantic-core passes when calling it. 369 370 Args: 371 self: The BaseModel instance. 372 context: The context. 373 """ 374 if getattr(self, '__pydantic_private__', None) is None: 375 pydantic_private = {} 376 for name, private_attr in self.__private_attributes__.items(): 377 # Avoid needlessly creating a new dict for the validated data: 378 if private_attr.default_factory_takes_validated_data: 379 default = private_attr.get_default( 380 call_default_factory=True, validated_data={**self.__dict__, **pydantic_private} 381 ) 382 else: 383 default = private_attr.get_default(call_default_factory=True) 384 if default is not PydanticUndefined: 385 pydantic_private[name] = default 386 object_setattr(self, '__pydantic_private__', pydantic_private)
This function is meant to behave like a BaseModel method to initialize private attributes.
It takes context as an argument since that's what pydantic-core passes when calling it.
Arguments:
- self: The BaseModel instance.
- context: The context.
Inherited Members
- pydantic.main.BaseModel
- BaseModel
- model_fields
- model_computed_fields
- model_extra
- model_fields_set
- model_construct
- model_copy
- model_dump
- model_dump_json
- model_json_schema
- model_parametrized_name
- model_rebuild
- model_validate
- model_validate_json
- model_validate_strings
- parse_file
- from_orm
- construct
- schema
- schema_json
- validate
- update_forward_refs