sqlmesh.integrations.github.cicd.command

  1from __future__ import annotations
  2
  3import logging
  4import traceback
  5
  6import click
  7
  8from sqlmesh.core.analytics import cli_analytics
  9from sqlmesh.core.console import set_console, MarkdownConsole
 10from sqlmesh.integrations.github.cicd.controller import (
 11    GithubCheckConclusion,
 12    GithubCheckStatus,
 13    GithubController,
 14    TestFailure,
 15)
 16from sqlmesh.utils.errors import CICDBotError, ConflictingPlanError, PlanError, LinterError
 17
 18logger = logging.getLogger(__name__)
 19
 20
 21@click.group(no_args_is_help=True)
 22@click.option(
 23    "--token",
 24    type=str,
 25    envvar="GITHUB_TOKEN",
 26    help="The Github Token to be used. Pass in `${{ secrets.GITHUB_TOKEN }}` if you want to use the one created by Github actions",
 27)
 28@click.option(
 29    "--full-logs",
 30    is_flag=True,
 31    help="Whether to print all logs in the Github Actions output or only in their relevant GA check",
 32)
 33@click.pass_context
 34def github(ctx: click.Context, token: str, full_logs: bool = False) -> None:
 35    """Github Action CI/CD Bot. See https://sqlmesh.readthedocs.io/en/stable/integrations/github/ for details"""
 36    # set a larger width because if none is specified, it auto-detects 80 characters when running in GitHub Actions
 37    # which can result in surprise newlines when outputting dates to backfill
 38    set_console(
 39        MarkdownConsole(
 40            width=1000, warning_capture_only=not full_logs, error_capture_only=not full_logs
 41        )
 42    )
 43    ctx.obj["github"] = GithubController(
 44        paths=ctx.obj["paths"],
 45        token=token,
 46        config=ctx.obj["config"],
 47    )
 48
 49
 50def _check_required_approvers(controller: GithubController) -> bool:
 51    controller.update_required_approval_check(status=GithubCheckStatus.IN_PROGRESS)
 52    if controller.has_required_approval:
 53        controller.update_required_approval_check(
 54            status=GithubCheckStatus.COMPLETED, conclusion=GithubCheckConclusion.SUCCESS
 55        )
 56        return True
 57    controller.update_required_approval_check(
 58        status=GithubCheckStatus.COMPLETED, conclusion=GithubCheckConclusion.FAILURE
 59    )
 60    return False
 61
 62
 63@github.command()
 64@click.pass_context
 65@cli_analytics
 66def check_required_approvers(ctx: click.Context) -> None:
 67    """Checks if a required approver has provided approval on the PR."""
 68    if not _check_required_approvers(ctx.obj["github"]):
 69        raise CICDBotError(
 70            "Required approver has not approved the PR. See Pull Requests Checks for more information."
 71        )
 72
 73
 74def _run_tests(controller: GithubController) -> bool:
 75    controller.update_test_check(status=GithubCheckStatus.IN_PROGRESS)
 76    try:
 77        result, _ = controller.run_tests()
 78        controller.update_test_check(
 79            status=GithubCheckStatus.COMPLETED,
 80            # Conclusion will be updated with final status based on test results
 81            conclusion=GithubCheckConclusion.NEUTRAL,
 82            result=result,
 83        )
 84        return result.wasSuccessful()
 85    except Exception:
 86        controller.update_test_check(
 87            status=GithubCheckStatus.COMPLETED,
 88            conclusion=GithubCheckConclusion.FAILURE,
 89            traceback=traceback.format_exc(),
 90        )
 91        return False
 92
 93
 94def _run_linter(controller: GithubController) -> bool:
 95    controller.update_linter_check(status=GithubCheckStatus.IN_PROGRESS)
 96    try:
 97        controller.run_linter()
 98    except LinterError:
 99        controller.update_linter_check(
100            status=GithubCheckStatus.COMPLETED,
101            conclusion=GithubCheckConclusion.FAILURE,
102        )
103        return False
104
105    controller.update_linter_check(
106        status=GithubCheckStatus.COMPLETED,
107        conclusion=GithubCheckConclusion.SUCCESS,
108    )
109
110    return True
111
112
113@github.command()
114@click.pass_context
115@cli_analytics
116def run_tests(ctx: click.Context) -> None:
117    """Runs the unit tests"""
118    if not _run_tests(ctx.obj["github"]):
119        raise CICDBotError("Failed to run tests. See Pull Requests Checks for more information.")
120
121
122def _update_pr_environment(controller: GithubController) -> bool:
123    controller.update_pr_environment_check(status=GithubCheckStatus.IN_PROGRESS)
124    try:
125        controller.update_pr_environment()
126        conclusion = controller.update_pr_environment_check(status=GithubCheckStatus.COMPLETED)
127        return conclusion is not None and conclusion.is_success
128    except Exception as e:
129        logger.exception("Error occurred when updating PR environment")
130        conclusion = controller.update_pr_environment_check(
131            status=GithubCheckStatus.COMPLETED, exception=e
132        )
133        return (
134            conclusion is not None
135            and not conclusion.is_failure
136            and not conclusion.is_action_required
137        )
138
139
140@github.command()
141@click.pass_context
142@cli_analytics
143def update_pr_environment(ctx: click.Context) -> None:
144    """Creates or updates the PR environments"""
145    if not _update_pr_environment(ctx.obj["github"]):
146        raise CICDBotError(
147            "Failed to update PR environment. See Pull Requests Checks for more information."
148        )
149
150
151def _gen_prod_plan(controller: GithubController) -> bool:
152    controller.update_prod_plan_preview_check(status=GithubCheckStatus.IN_PROGRESS)
153    try:
154        plan_summary = controller.get_plan_summary(controller.prod_plan)
155        controller.update_prod_plan_preview_check(
156            status=GithubCheckStatus.COMPLETED,
157            conclusion=GithubCheckConclusion.SUCCESS,
158            summary=plan_summary,
159        )
160        return bool(plan_summary)
161    except Exception as e:
162        logger.exception("Error occurred generating prod plan")
163        controller.update_prod_plan_preview_check(
164            status=GithubCheckStatus.COMPLETED,
165            conclusion=GithubCheckConclusion.FAILURE,
166            summary=str(e),
167        )
168        return False
169
170
171@github.command()
172@click.pass_context
173@cli_analytics
174def gen_prod_plan(ctx: click.Context) -> None:
175    """Generates the production plan"""
176    controller = ctx.obj["github"]
177    controller.update_prod_plan_preview_check(status=GithubCheckStatus.IN_PROGRESS)
178    if not _gen_prod_plan(controller):
179        raise CICDBotError(
180            "Failed to generate production plan. See Pull Requests Checks for more information."
181        )
182
183
184def _deploy_production(controller: GithubController) -> bool:
185    controller.update_prod_environment_check(status=GithubCheckStatus.IN_PROGRESS)
186    try:
187        controller.deploy_to_prod()
188        controller.update_prod_environment_check(
189            status=GithubCheckStatus.COMPLETED, conclusion=GithubCheckConclusion.SUCCESS
190        )
191        controller.try_merge_pr()
192        controller.try_invalidate_pr_environment()
193        return True
194    except ConflictingPlanError as e:
195        controller.update_prod_environment_check(
196            status=GithubCheckStatus.COMPLETED,
197            conclusion=GithubCheckConclusion.SKIPPED,
198            skip_reason=str(e),
199        )
200        return False
201    except PlanError as e:
202        controller.update_prod_environment_check(
203            status=GithubCheckStatus.COMPLETED,
204            conclusion=GithubCheckConclusion.ACTION_REQUIRED,
205            plan_error=e,
206        )
207        return False
208    except Exception:
209        controller.update_prod_environment_check(
210            status=GithubCheckStatus.COMPLETED, conclusion=GithubCheckConclusion.FAILURE
211        )
212        return False
213
214
215@github.command()
216@click.pass_context
217@cli_analytics
218def deploy_production(ctx: click.Context) -> None:
219    """Deploys the production environment"""
220    if not _deploy_production(ctx.obj["github"]):
221        raise CICDBotError(
222            "Failed to deploy to production. See Pull Requests Checks for more information."
223        )
224
225
226def _run_all(controller: GithubController) -> None:
227    click.echo(f"SQLMesh Version: {controller.version_info}")
228
229    has_required_approval = False
230    is_auto_deploying_prod = (
231        controller.deploy_command_enabled or controller.do_required_approval_check
232    ) and controller.pr_targets_prod_branch
233    if controller.is_comment_added:
234        if not controller.deploy_command_enabled:
235            # We aren't using commands so we can just return
236            return
237        command = controller.get_command_from_comment()
238        if command.is_invalid:
239            # Probably a comment unrelated to SQLMesh so we do nothing
240            return
241        if command.is_deploy_prod:
242            has_required_approval = True
243        else:
244            raise CICDBotError(f"Unsupported command: {command}")
245    controller.update_linter_check(status=GithubCheckStatus.QUEUED)
246    controller.update_pr_environment_check(status=GithubCheckStatus.QUEUED)
247    controller.update_prod_plan_preview_check(status=GithubCheckStatus.QUEUED)
248    controller.update_test_check(status=GithubCheckStatus.QUEUED)
249    if is_auto_deploying_prod:
250        controller.update_prod_environment_check(status=GithubCheckStatus.QUEUED)
251    linter_passed = _run_linter(controller)
252    tests_passed = _run_tests(controller)
253    if controller.do_required_approval_check:
254        if has_required_approval:
255            controller.update_required_approval_check(
256                status=GithubCheckStatus.COMPLETED, conclusion=GithubCheckConclusion.SKIPPED
257            )
258        else:
259            controller.update_required_approval_check(status=GithubCheckStatus.QUEUED)
260            has_required_approval = _check_required_approvers(controller)
261    if not tests_passed or not linter_passed:
262        controller.update_pr_environment_check(
263            status=GithubCheckStatus.COMPLETED,
264            exception=LinterError("") if not linter_passed else TestFailure(),
265        )
266        controller.update_prod_plan_preview_check(
267            status=GithubCheckStatus.COMPLETED,
268            conclusion=GithubCheckConclusion.SKIPPED,
269            summary="Linter or Unit Test(s) failed so skipping creating prod plan",
270        )
271        if is_auto_deploying_prod:
272            controller.update_prod_environment_check(
273                status=GithubCheckStatus.COMPLETED,
274                conclusion=GithubCheckConclusion.SKIPPED,
275                skip_reason="Linter or Unit Test(s) failed so skipping deploying to production",
276            )
277
278        raise CICDBotError(
279            "Linter or Unit Test(s) failed. See Pull Requests Checks for more information."
280        )
281
282    pr_environment_updated = _update_pr_environment(controller)
283    prod_plan_generated = False
284    if pr_environment_updated:
285        prod_plan_generated = _gen_prod_plan(controller)
286    else:
287        controller.update_prod_plan_preview_check(
288            status=GithubCheckStatus.COMPLETED, conclusion=GithubCheckConclusion.SKIPPED
289        )
290    deployed_to_prod = False
291    if has_required_approval and prod_plan_generated and controller.pr_targets_prod_branch:
292        deployed_to_prod = _deploy_production(controller)
293    elif is_auto_deploying_prod:
294        if controller.deploy_command_enabled and not has_required_approval:
295            skip_reason = "Skipped Deploying to Production because a `/deploy` command has not been detected yet"
296        elif controller.do_required_approval_check and not has_required_approval:
297            skip_reason = (
298                "Skipped Deploying to Production because a required approver has not approved"
299            )
300        elif not pr_environment_updated:
301            skip_reason = (
302                "Skipped Deploying to Production because the PR environment was not updated"
303            )
304        elif not prod_plan_generated:
305            skip_reason = (
306                "Skipped Deploying to Production because the production plan could not be generated"
307            )
308        else:
309            skip_reason = "Skipped Deploying to Production for an unknown reason"
310        controller.update_prod_environment_check(
311            status=GithubCheckStatus.COMPLETED,
312            conclusion=GithubCheckConclusion.SKIPPED,
313            skip_reason=skip_reason,
314        )
315    if (
316        not pr_environment_updated
317        or not prod_plan_generated
318        or (has_required_approval and controller.pr_targets_prod_branch and not deployed_to_prod)
319    ):
320        raise CICDBotError(
321            "A step of the run-all check failed. See Pull Requests Checks for more information."
322        )
323
324
325@github.command()
326@click.pass_context
327@cli_analytics
328def run_all(ctx: click.Context) -> None:
329    """Runs all the commands in the correct order."""
330    return _run_all(ctx.obj["github"])