Merge branch 'main' into main

Author: codetalker-ai

docs/cloud/features/scheduler/scheduler.md

@@ -167,4 +167,77 @@ Tobiko Cloud automatically manages Python dependencies of your Python macros and
 
 SQLMesh automatically infers which Python libraries are used by statically analyzing the code of your models and macros.
 
-For fine-grained control, dependencies can be specified, pinned, or excluded using the `sqlmesh-requirements.lock` file. See the [Python library dependencies](../../../guides/configuration.md#python-library-dependencies) section in the SQLMesh configuration guide for more information.
+For fine-grained control, dependencies can be specified, pinned, or excluded using the `sqlmesh-requirements.lock` file. See the [Python library dependencies](../../../guides/configuration.md#python-library-dependencies) section in the SQLMesh configuration guide for more information.
+
+## Secret Manager
+
+Tobiko Cloud provides a secrets manager where you can define environment variables for your project's Python models.
+
+These variables are most commonly used to provide sensitive information to Python models, such as API keys or other credentials.
+
+Secret values are encrypted at rest and only available in the environment of your running Python models.
+
+!!! note "Cloud Scheduler Only"
+
+    Secrets from the secret manager do not load into hybrid executors. They are only used for cloud scheduler executors.
+
+Secret names have two restrictions - they must:
+
+- Start with a letter or an underscore
+- Only include letters, numbers, and underscores (no spaces or other symbols)
+
+Secret values have no limits or restrictions. We recommend base64 encoding any secrets that contain binary data.
+
+### Defining secrets
+
+Define a secret on the Secrets page, accessible via the Settings section in Tobiko Cloud's left side navigation bar.
+
+The Secrets page has a single panel you use to create a new secret, edit the value of an existing secret, or remove an existing secret. You cannot view the value of any existing secret.
+
+In this example, only one secret has been defined: `MY_SECRET`. Update its value by entering a new value in the Secret field and clicking the `Update` button, or delete it by clicking the `Remove` button.
+
+![secrets_panel](./scheduler/secrets.png)
+
+
+### Python Model Example
+
+This Python model demonstrates how to read the `MY_SECRET` secret from an environment variable.
+
+!!! danger "Protecting Secrets"
+
+    Only read environment variables from inside a Python model's `execute` function definition (not in the global scope).
+
+    If the variable is read in the global scope, SQLMesh will load the value from *your local system* when it renders the Python model instead of loading it at runtime on our executors.
+
+    This could expose sensitive information or embed an incorrect local value in the rendered model.
+
+```python linenums="1"
+import os
+import pandas as pd
+import typing as t
+from datetime import datetime
+
+from sqlmesh import ExecutionContext, model
+
+# DO NOT read environment variables here.
+# Only inside the `execute` function definition!
+
+@model(
+    "my_model.name",
+    columns={
+        "column_name": "int",
+    },
+)
+def execute(
+    context: ExecutionContext,
+    start: datetime,
+    end: datetime,
+    execution_time: datetime,
+    **kwargs: t.Any,
+) -> pd.DataFrame:
+
+    # Read a secret from the MY_SECRET environment variable
+    my_secret = os.environ["MY_SECRET"]
+
+    ...
+```

docs/cloud/features/scheduler/scheduler/secrets.png

@@ -28,7 +28,6 @@
 from sqlmesh.utils.errors import ConfigError
 from sqlmesh.utils.jinja import (
     JinjaMacroRegistry,
-    extract_macro_references_and_variables,
     make_jinja_registry,
 )
 
@@ -240,11 +239,11 @@ def _load_requirements(self) -> t.Tuple[t.Dict[str, str], t.Set[str]]:
     def _load_environment_statements(self, macros: MacroRegistry) -> t.List[EnvironmentStatements]:
         """Loads dbt's on_run_start, on_run_end hooks into sqlmesh's before_all, after_all statements respectively."""
 
-        environment_statements: t.List[EnvironmentStatements] = []
+        hooks_by_package_name: t.Dict[str, EnvironmentStatements] = {}
+        project_names: t.Set[str] = set()
         dialect = self.config.dialect
         for project in self._load_projects():
             context = project.context
-            hooks_by_package_name: t.Dict[str, EnvironmentStatements] = {}
             for package_name, package in project.packages.items():
                 context.set_and_render_variables(package.variables, package_name)
                 on_run_start: t.List[str] = [
@@ -256,18 +255,14 @@ def _load_environment_statements(self, macros: MacroRegistry) -> t.List[Environm
                     for on_run_hook in sorted(package.on_run_end.values(), key=lambda h: h.index)
                 ]
 
-                if statements := on_run_start + on_run_end:
-                    jinja_references, used_variables = extract_macro_references_and_variables(
-                        *statements
-                    )
+                if on_run_start or on_run_end:
+                    dependencies = Dependencies()
+                    for hook in [*package.on_run_start.values(), *package.on_run_end.values()]:
+                        dependencies = dependencies.union(hook.dependencies)
 
-                    statements_context = context.context_for_dependencies(
-                        Dependencies(
-                            variables=used_variables,
-                        )
-                    )
+                    statements_context = context.context_for_dependencies(dependencies)
                     jinja_registry = make_jinja_registry(
-                        statements_context.jinja_macros, package_name, jinja_references
+                        statements_context.jinja_macros, package_name, set(dependencies.macros)
                     )
                     jinja_registry.add_globals(statements_context.jinja_globals)
 
@@ -283,15 +278,15 @@ def _load_environment_statements(self, macros: MacroRegistry) -> t.List[Environm
                         python_env={},
                         jinja_macros=jinja_registry,
                     )
-                # Project hooks should be executed first and then rest of the packages
-                environment_statements = [
-                    statements
-                    for _, statements in sorted(
-                        hooks_by_package_name.items(),
-                        key=lambda item: 0 if item[0] == context.project_name else 1,
-                    )
-                ]
-        return environment_statements
+                    project_names.add(package_name)
+
+        return [
+            statements
+            for _, statements in sorted(
+                hooks_by_package_name.items(),
+                key=lambda item: 0 if item[0] in project_names else 1,
+            )
+        ]
 
     def _compute_yaml_max_mtime_per_subfolder(self, root: Path) -> t.Dict[Path, float]:
         if not root.is_dir():

sqlmesh/dbt/loader.py

@@ -292,13 +292,24 @@ def _load_on_run_start_end(self) -> None:
                 sql = node.raw_code if DBT_VERSION >= (1, 3) else node.raw_sql  # type: ignore
                 node_name = node.name
                 node_path = Path(node.original_file_path)
+
+                dependencies = Dependencies(
+                    macros=_macro_references(self._manifest, node),
+                    refs=_refs(node),
+                    sources=_sources(node),
+                )
+                dependencies = dependencies.union(self._extra_dependencies(sql, node.package_name))
+                dependencies = dependencies.union(
+                    self._flatten_dependencies_from_macros(dependencies.macros, node.package_name)
+                )
+
                 if "on-run-start" in node.tags:
                     self._on_run_start_per_package[node.package_name][node_name] = HookConfig(
-                        sql=sql, index=node.index or 0, path=node_path
+                        sql=sql, index=node.index or 0, path=node_path, dependencies=dependencies
                     )
                 else:
                     self._on_run_end_per_package[node.package_name][node_name] = HookConfig(
-                        sql=sql, index=node.index or 0, path=node_path
+                        sql=sql, index=node.index or 0, path=node_path, dependencies=dependencies
                     )
 
     @property

sqlmesh/dbt/manifest.py

@@ -34,6 +34,7 @@ class HookConfig(PydanticModel):
     sql: str
     index: int
     path: Path
+    dependencies: Dependencies
 
 
 class Package(PydanticModel):