Merge branch 'main' into add_fabric_warehouse

Author: fresioAS

docs/concepts/models/python_models.md

@@ -163,15 +163,15 @@ def execute(
 ) -> pd.DataFrame:
 
     # pre-statement
-    context.fetchdf("SET GLOBAL parameter = 'value';")
+    context.engine_adapter.execute("SET GLOBAL parameter = 'value';")
 
     # post-statement requires using `yield` instead of `return`
     yield pd.DataFrame([
         {"id": 1, "name": "name"}
     ])
 
     # post-statement
-    context.fetchdf("CREATE INDEX idx ON example.pre_post_statements (id);")
+    context.engine_adapter.execute("CREATE INDEX idx ON example.pre_post_statements (id);")
 ```
 
 ## Optional on-virtual-update statements

docs/guides/configuration.md

@@ -149,6 +149,16 @@ Because the VSCode extension establishes a long-running process connected to the
 
 Therefore, we do not recommend using DuckDB as a state store with the VSCode extension.
 
+### Environment variables
+
+The VSCode extension is based on a [language server](https://en.wikipedia.org/wiki/Language_Server_Protocol) that runs in the background as a separate process. When the VSCode extension starts the background language server, the server inherits environment variables from the environment where you started VSCode. The server does *not* inherit environment variables from your terminal instance in VSCode, so it may not have access to variables you use when calling SQLMesh from the CLI.
+
+If you have environment variables that are needed by the context and the language server, you can use one of these approaches to pass variables to the language server:
+
+- Open VSCode from a terminal that has the variables set
+- Use environment variables pulled from somewhere else dynamically (e.g. a `.env` file) in your config 
+- Set the environment variables in the python environment that the extension uses. You can find detailed instructions [here](https://code.visualstudio.com/docs/python/environments#_environment-variables)
+
 ### Python environment woes
 
 The most common problem is the extension not using the correct Python interpreter.

docs/guides/vscode.md

@@ -9,7 +9,7 @@ SQLMesh supports integrations with the following tools:
 * [Kestra](https://kestra.io/plugins/plugin-sqlmesh/tasks/cli/io.kestra.plugin.sqlmesh.cli.sqlmeshcli)
 
 ## Execution engines
-SQLMesh supports the following execution engines for running SQLMesh projects:
+SQLMesh supports the following execution engines for running SQLMesh projects (engine `type` in parentheses - example usage: `pip install "sqlmesh[databricks]"`):
 
 * [Athena](./engines/athena.md)
 * [Azure SQL](./engines/azuresql.md)
@@ -26,4 +26,4 @@ SQLMesh supports the following execution engines for running SQLMesh projects:
 * [Redshift](./engines/redshift.md)
 * [Snowflake](./engines/snowflake.md)
 * [Spark](./engines/spark.md)
-* [Trino](./engines/trino.md)
+* [Trino](./engines/trino.md)

docs/integrations/overview.md

@@ -267,15 +267,17 @@ Options:
 ## init
 
 ```
-Usage: sqlmesh init [OPTIONS] [SQL_DIALECT]
+Usage: sqlmesh init [OPTIONS] [ENGINE]
 
   Create a new SQLMesh repository.
 
 Options:
-  -t, --template TEXT  Project template. Supported values: dbt,
-                       dlt, default, empty.
+  -t, --template TEXT  Project template. Supported values: dbt, dlt, default,
+                       empty.
   --dlt-pipeline TEXT  DLT pipeline for which to generate a SQLMesh project.
-                       For use with dlt template.
+                       Use alongside template: dlt
+  --dlt-path TEXT      The directory where the DLT pipeline resides. Use
+                       alongside template: dlt
   --help               Show this message and exit.
 ```
 
@@ -419,19 +421,31 @@ Usage: sqlmesh render [OPTIONS] MODEL
   Render a model's query, optionally expanding referenced models.
 
 Options:
-  -s, --start TEXT       The start datetime of the interval for which this
-                         command will be applied.
-  -e, --end TEXT         The end datetime of the interval for which this
-                         command will be applied.
-  --execution-time TEXT  The execution time (defaults to now).
-  --expand TEXT          Whether or not to expand materialized models
-                         (defaults to False). If True, all referenced models
-                         are expanded as raw queries. Multiple model names can
-                         also be specified, in which case only they will be
-                         expanded as raw queries.
-  --dialect TEXT         The SQL dialect to render the query as.
-  --no-format            Disable fancy formatting of the query.
-  --help                 Show this message and exit.
+  -s, --start TEXT            The start datetime of the interval for which
+                              this command will be applied.
+  -e, --end TEXT              The end datetime of the interval for which this
+                              command will be applied.
+  --execution-time TEXT       The execution time (defaults to now).
+  --expand TEXT               Whether or not to expand materialized models
+                              (defaults to False). If True, all referenced
+                              models are expanded as raw queries. Multiple
+                              model names can also be specified, in which case
+                              only they will be expanded as raw queries.
+  --dialect TEXT              The SQL dialect to render the query as.
+  --no-format                 Disable fancy formatting of the query.
+  --max-text-width INTEGER    The max number of characters in a segment before
+                              creating new lines in pretty mode.
+  --leading-comma             Determines whether or not the comma is leading
+                              or trailing in select expressions. Default is
+                              trailing.
+  --normalize-functions TEXT  Whether or not to normalize all function names.
+                              Possible values are: 'upper', 'lower'
+  --indent INTEGER            Determines the indentation size in a formatted
+                              string.
+  --pad INTEGER               Determines the pad size in a formatted string.
+  --normalize                 Whether or not to normalize identifiers to
+                              lowercase.
+  --help                      Show this message and exit.
 ```
 
 ## rewrite

docs/quickstart/cli.md

@@ -29,9 +29,9 @@ import sqlmesh
 If desired, you can create the [quickstart example project](../quick_start.md) with the Python `init_example_project` function. The function requires a default SQL dialect for the project's models; this example uses `snowflake`:
 
 ```python
-from sqlmesh.cli.example_project import init_example_project
+from sqlmesh.cli.project_init import init_example_project
 
-init_example_project("path_to_project_directory", dialect="snowflake")
+init_example_project("path_to_project_directory", engine_type="snowflake")
 ```
 
 Alternatively, create the project with a notebook magic:
@@ -201,6 +201,9 @@ options:
 ```
 %render [--start START] [--end END] [--execution-time EXECUTION_TIME]
               [--expand EXPAND] [--dialect DIALECT] [--no-format]
+              [--normalize] [--pad PAD] [--indent INDENT]
+              [--normalize-functions NORMALIZE_FUNCTIONS] [--leading-comma]
+              [--max-text-width MAX_TEXT_WIDTH]
               model
 
 Renders a model's query, optionally expanding referenced models.
@@ -220,6 +223,17 @@ options:
                         models are expanded as raw queries.
   --dialect DIALECT     SQL dialect to render.
   --no-format           Disable fancy formatting of the query.
+  --normalize           Whether or not to normalize identifiers to lowercase.
+  --pad PAD             Determines the pad size in a formatted string.
+  --indent INDENT       Determines the indentation size in a formatted string.
+  --normalize-functions NORMALIZE_FUNCTIONS
+                        Whether or not to normalize all function names.
+                        Possible values are: 'upper', 'lower'
+  --leading-comma       Determines whether or not the comma is leading or
+                        trailing in select expressions. Default is trailing.
+  --max-text-width MAX_TEXT_WIDTH
+                        The max number of characters in a segment before
+                        creating new lines in pretty mode.
 ```
 
 #### dag

docs/quickstart/cli/cli-quickstart_duckdb-tables.png

@@ -128,12 +128,17 @@
 )
 
 
-environment_suffix_config = Config(
+environment_suffix_table_config = Config(
     default_connection=DuckDBConnectionConfig(),
     model_defaults=model_defaults,
     environment_suffix_target=EnvironmentSuffixTarget.TABLE,
 )
 
+environment_suffix_catalog_config = environment_suffix_table_config.model_copy(
+    update={
+        "environment_suffix_target": EnvironmentSuffixTarget.CATALOG,
+    }
+)
 
 CATALOGS = {
     "in_memory": ":memory:",