Prepare for prerelease verification (#777)

dbt/adapters/databricks/connections.py

@@ -818,7 +818,6 @@ def set_connection_name(
         'connection_named', called by 'connection_for(node)'.
         Creates a connection for this thread if one doesn't already
         exist, and will rename an existing connection."""
-
         self._cleanup_idle_connections()
 
         conn_name: str = "master" if name is None else name

pytest.ini

@@ -8,3 +8,7 @@ testpaths =
     tests/unit
     tests/integration
     tests/functional
+markers =
+    external: mark test as requiring an external location
+    python: mark test as running a python model
+    dlt: mark test as running a DLT model

tests/functional/adapter/basic/test_incremental.py

@@ -24,6 +24,7 @@ class TestIncrementalDeltaNotSchemaChange(BaseIncrementalNotSchemaChange):
     pass
 
 
+@pytest.mark.external
 @pytest.mark.skip_profile("databricks_uc_cluster", "databricks_cluster")
 class TestIncrementalParquet(BaseIncremental):
     @pytest.fixture(scope="class")
@@ -50,6 +51,7 @@ def project_config_update(self):
         }
 
 
+@pytest.mark.external
 @pytest.mark.skip_profile("databricks_uc_cluster", "databricks_cluster")
 class TestIncrementalCSV(BaseIncremental):
     @pytest.fixture(scope="class")

tests/functional/adapter/incremental/test_incremental_clustering.py

@@ -36,6 +36,7 @@ def test_changing_cluster_by(self, project):
                 assert False
 
 
+@pytest.mark.python
 @pytest.mark.skip_profile("databricks_cluster")
 class TestIncrementalPythonLiquidClustering:
     @pytest.fixture(scope="class")

tests/functional/adapter/incremental/test_incremental_strategies.py

@@ -42,6 +42,7 @@ class TestAppendDelta(AppendBase):
     pass
 
 
+@pytest.mark.external
 @pytest.mark.skip_profile("databricks_uc_cluster", "databricks_cluster")
 class TestAppendParquet(AppendBase):
     @pytest.fixture(scope="class")
@@ -118,6 +119,7 @@ def test_incremental(self, project):
         util.check_relations_equal(project.adapter, ["overwrite_model", "upsert_expected"])
 
 
+@pytest.mark.external
 @pytest.mark.skip("This test is not repeatable due to external location")
 class TestInsertOverwriteParquet(InsertOverwriteBase):
     @pytest.fixture(scope="class")
@@ -132,6 +134,7 @@ def project_config_update(self):
         }
 
 
+@pytest.mark.external
 @pytest.mark.skip("This test is not repeatable due to external location")
 class TestInsertOverwriteWithPartitionsParquet(InsertOverwriteBase):
     @pytest.fixture(scope="class")

tests/functional/adapter/incremental/test_incremental_tags.py

@@ -28,6 +28,7 @@ def test_changing_tags(self, project):
         assert results_dict == {"c": "e", "d": "f"}
 
 
+@pytest.mark.python
 @pytest.mark.skip_profile("databricks_cluster")
 class TestIncrementalPythonTags:
     @pytest.fixture(scope="class")

tests/functional/adapter/incremental/test_incremental_tblproperties.py

@@ -27,6 +27,7 @@ def test_changing_tblproperties(self, project):
         assert results_dict["d"] == "f"
 
 
+@pytest.mark.python
 @pytest.mark.skip_profile("databricks_cluster")
 class TestIncrementalPythonTblproperties:
     @pytest.fixture(scope="class")

tests/functional/adapter/materialized_view_tests/test_basic.py

@@ -27,6 +27,7 @@ def query_relation_type(project, relation: BaseRelation) -> Optional[str]:
         return fixtures.query_relation_type(project, relation)
 
 
+@pytest.mark.dlt
 @pytest.mark.skip_profile("databricks_cluster", "databricks_uc_cluster")
 class TestMaterializedViews(TestMaterializedViewsMixin, MaterializedViewBasic):
     def test_table_replaces_materialized_view(self, project, my_materialized_view):

tests/functional/adapter/materialized_view_tests/test_changes.py

@@ -80,20 +80,23 @@ def query_relation_type(project, relation: BaseRelation) -> Optional[str]:
         return fixtures.query_relation_type(project, relation)
 
 
+@pytest.mark.dlt
 @pytest.mark.skip_profile("databricks_cluster", "databricks_uc_cluster")
 class TestMaterializedViewApplyChanges(
     MaterializedViewChangesMixin, MaterializedViewChangesApplyMixin
 ):
     pass
 
 
+@pytest.mark.dlt
 @pytest.mark.skip_profile("databricks_cluster", "databricks_uc_cluster")
 class TestMaterializedViewContinueOnChanges(
     MaterializedViewChangesMixin, MaterializedViewChangesContinueMixin
 ):
     pass
 
 
+@pytest.mark.dlt
 @pytest.mark.skip_profile("databricks_cluster", "databricks_uc_cluster")
 class TestMaterializedViewFailOnChanges(
     MaterializedViewChangesMixin, MaterializedViewChangesFailMixin

tests/functional/adapter/persist_docs/test_persist_docs.py

@@ -245,6 +245,7 @@ def test_quoted_column_comments(self, adapter, table_relation):
                 break
 
 
+@pytest.mark.external
 # Skipping UC Cluster to ensure these tests don't fail due to overlapping resources
 @pytest.mark.skip_profile("databricks_uc_cluster")
 class TestPersistDocsWithSeeds:

tests/functional/adapter/python_model/test_python_model.py

@@ -9,16 +9,19 @@
 from tests.functional.adapter.python_model import fixtures as override_fixtures
 
 
+@pytest.mark.python
 @pytest.mark.skip_profile("databricks_uc_sql_endpoint")
 class TestPythonModel(BasePythonModelTests):
     pass
 
 
+@pytest.mark.python
 @pytest.mark.skip_profile("databricks_uc_sql_endpoint")
 class TestPythonIncrementalModel(BasePythonIncrementalTests):
     pass
 
 
+@pytest.mark.python
 @pytest.mark.skip_profile("databricks_uc_sql_endpoint")
 class TestChangingSchema:
     @pytest.fixture(scope="class")
@@ -42,6 +45,7 @@ def test_changing_schema_with_log_validation(self, project, logs_dir):
             assert "Execution status: OK in" in log
 
 
+@pytest.mark.python
 @pytest.mark.skip_profile("databricks_uc_sql_endpoint")
 class TestChangingSchemaIncremental:
     @pytest.fixture(scope="class")
@@ -60,6 +64,7 @@ def test_changing_schema_via_incremental(self, project):
         util.check_relations_equal(project.adapter, ["incremental_model", "expected_incremental"])
 
 
+@pytest.mark.python
 @pytest.mark.skip_profile("databricks_cluster", "databricks_uc_cluster")
 class TestSpecifyingHttpPath(BasePythonModelTests):
     @pytest.fixture(scope="class")
@@ -73,6 +78,8 @@ def models(self):
         }
 
 
+@pytest.mark.python
+@pytest.mark.external
 @pytest.mark.skip_profile("databricks_cluster", "databricks_uc_sql_endpoint")
 class TestComplexConfig:
     @pytest.fixture(scope="class")

tests/functional/adapter/python_model/test_spark.py

@@ -7,6 +7,7 @@
 )
 
 
+@pytest.mark.python
 @pytest.mark.skip_profile("databricks_uc_sql_endpoint")
 class TestPySpark(BasePySparkTests):
     @pytest.fixture(scope="class")

tests/functional/adapter/simple_copy/test_simple_copy.py

@@ -5,6 +5,7 @@
 
 
 # Tests with materialized_views, which only works for SQL Warehouse
+@pytest.mark.dlt
 @pytest.mark.skip_profile("databricks_cluster", "databricks_uc_cluster")
 class TestSimpleCopyBase(SimpleCopyBase):
     pass

tests/functional/adapter/streaming_tables/test_st_basic.py

@@ -12,6 +12,7 @@
 from tests.functional.adapter.streaming_tables import fixtures
 
 
+@pytest.mark.dlt
 @pytest.mark.skip_profile("databricks_cluster", "databricks_uc_cluster")
 class TestStreamingTablesBasic:
     @staticmethod

tests/functional/adapter/streaming_tables/test_st_changes.py

@@ -123,6 +123,7 @@ def test_full_refresh_occurs_with_changes(self, project, my_streaming_table):
         util.assert_message_in_logs(f"Applying REPLACE to: {my_streaming_table}", logs)
 
 
+@pytest.mark.dlt
 @pytest.mark.skip_profile("databricks_cluster", "databricks_uc_cluster")
 class TestStreamingTableChangesApply(StreamingTableChanges):
     @pytest.fixture(scope="class")
@@ -153,6 +154,7 @@ def test_change_is_applied_via_replace(self, project, my_streaming_table):
         util.assert_message_in_logs(f"Applying REPLACE to: {my_streaming_table}", logs)
 
 
+@pytest.mark.dlt
 @pytest.mark.skip_profile("databricks_cluster", "databricks_uc_cluster")
 class TestStreamingTableChangesContinue(StreamingTableChanges):
     @pytest.fixture(scope="class")
@@ -193,6 +195,7 @@ def test_change_is_not_applied_via_replace(self, project, my_streaming_table):
         util.assert_message_in_logs(f"Applying REPLACE to: {my_streaming_table}", logs, False)
 
 
+@pytest.mark.dlt
 @pytest.mark.skip_profile("databricks_cluster", "databricks_uc_cluster")
 class TestStreamingTableChangesFail(StreamingTableChanges):
     @pytest.fixture(scope="class")

tests/functional/adapter/tags/test_databricks_tags.py

@@ -36,6 +36,7 @@ def models(self):
         return {"tags.sql": fixtures.tags_sql.replace("table", "incremental")}
 
 
+@pytest.mark.python
 @pytest.mark.skip_profile("databricks_cluster")
 class TestPythonTags(TestTags):
     @pytest.fixture(scope="class")